hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-20 14:34:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16117 from github/redsun82/kotlin

Browse Source 
Kotlin: build extractor with bazel
This commit is contained in:
Paolo Tranquilli
2024-06-04 20:13:00 +02:00
committed by GitHub
parent 60970ff015 b7e16ca55d
commit ad2eacfd2f
66 changed files with 2835 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
.gitattributes vendored
View File

@@ -50,36 +50,37 @@
*.dll -text
*.pdb -text
java/ql/test/stubs/**/*.java linguist-generated=true
java/ql/test/experimental/stubs/**/*.java linguist-generated=true
/java/ql/test/stubs/**/*.java linguist-generated=true
/java/ql/test/experimental/stubs/**/*.java linguist-generated=true
/java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text
# Force git not to modify line endings for go or html files under the go/ql directory
go/ql/**/*.go -text
go/ql/**/*.html -text
/go/ql/**/*.go -text
/go/ql/**/*.html -text
# Force git not to modify line endings for go dbschemes
go/*.dbscheme -text
/go/*.dbscheme -text
# Preserve unusual line ending from codeql-go merge
go/extractor/opencsv/CSVReader.java -text
/go/extractor/opencsv/CSVReader.java -text
# For some languages, upgrade script testing references really old dbscheme
# files from legacy upgrades that have CRLF line endings. Since upgrade
# resolution relies on object hashes, we must suppress line ending conversion
# for those testing dbscheme files.
*/ql/lib/upgrades/initial/*.dbscheme -text
/*/ql/lib/upgrades/initial/*.dbscheme -text
# Auto-generated modeling for Python
python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
/python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
# auto-generated bazel lock file
ruby/extractor/cargo-bazel-lock.json linguist-generated=true
ruby/extractor/cargo-bazel-lock.json -merge
/ruby/extractor/cargo-bazel-lock.json linguist-generated=true
/ruby/extractor/cargo-bazel-lock.json -merge
# auto-generated files for the C# build
csharp/paket.lock linguist-generated=true
# needs eol=crlf, as `paket` touches this file and saves it als crlf
csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf
csharp/paket.main.bzl linguist-generated=true
csharp/paket.main_extension.bzl linguist-generated=true
/csharp/paket.lock linguist-generated=true
# needs eol=crlf, as `paket` touches this file and saves it as crlf
/csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf
/csharp/paket.main.bzl linguist-generated=true
/csharp/paket.main_extension.bzl linguist-generated=true
# ripunzip tool
/misc/bazel/internal/ripunzip/ripunzip-* filter=lfs diff=lfs merge=lfs -text

28
.github/workflows/kotlin-build.yml vendored Normal file
View File

@@ -0,0 +1,28 @@
name: "Kotlin Build"
on:
pull_request:
paths:
- "java/kotlin-extractor/**"
- "misc/bazel/**"
- "misc/codegen/**"
- "*.bazel*"
- .github/workflows/kotlin-build.yml
branches:
- main
- rc/*
- codeql-cli-*
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: |
bazel query //java/kotlin-extractor/...
# only build the default version as a quick check that we can build from `codeql`
# the full official build will be checked by QLucie
bazel build //java/kotlin-extractor

2
.lfsconfig
View File

@@ -2,4 +2,6 @@
# codeql is publicly forked by many users, and we don't want any LFS file polluting their working
# copies. We therefore exclude everything by default.
# For files required by bazel builds, use rules in `misc/bazel/lfs.bzl` to download them on demand.
# we go for `fetchinclude` to something not exsiting rather than `fetchexclude = *` because the
# former is easier to override (with `git -c` or a local git config) to fetch something specific
fetchinclude = /nothing

46
MODULE.bazel
View File

@@ -22,6 +22,7 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0")
bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl")
bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
bazel_dep(name = "fmt", version = "10.0.0")
bazel_dep(name = "rules_kotlin", version = "1.9.4-codeql.1")
bazel_dep(name = "gazelle", version = "0.36.0")
bazel_dep(name = "rules_dotnet", version = "0.15.1")
bazel_dep(name = "googletest", version = "1.14.0.bcr.1")
@@ -65,6 +66,51 @@ node.toolchain(
)
use_repo(node, "nodejs", "nodejs_toolchains")
kotlin_extractor_deps = use_extension("//java/kotlin-extractor:deps.bzl", "kotlin_extractor_deps")
# following list can be kept in sync by running `bazel mod tidy` in `codeql`
use_repo(
kotlin_extractor_deps,
"codeql_kotlin_defaults",
"codeql_kotlin_embeddable",
"kotlin-compiler-1.5.0",
"kotlin-compiler-1.5.10",
"kotlin-compiler-1.5.20",
"kotlin-compiler-1.5.30",
"kotlin-compiler-1.6.0",
"kotlin-compiler-1.6.20",
"kotlin-compiler-1.7.0",
"kotlin-compiler-1.7.20",
"kotlin-compiler-1.8.0",
"kotlin-compiler-1.9.0-Beta",
"kotlin-compiler-1.9.20-Beta",
"kotlin-compiler-2.0.0-RC1",
"kotlin-compiler-embeddable-1.5.0",
"kotlin-compiler-embeddable-1.5.10",
"kotlin-compiler-embeddable-1.5.20",
"kotlin-compiler-embeddable-1.5.30",
"kotlin-compiler-embeddable-1.6.0",
"kotlin-compiler-embeddable-1.6.20",
"kotlin-compiler-embeddable-1.7.0",
"kotlin-compiler-embeddable-1.7.20",
"kotlin-compiler-embeddable-1.8.0",
"kotlin-compiler-embeddable-1.9.0-Beta",
"kotlin-compiler-embeddable-1.9.20-Beta",
"kotlin-compiler-embeddable-2.0.0-RC1",
"kotlin-stdlib-1.5.0",
"kotlin-stdlib-1.5.10",
"kotlin-stdlib-1.5.20",
"kotlin-stdlib-1.5.30",
"kotlin-stdlib-1.6.0",
"kotlin-stdlib-1.6.20",
"kotlin-stdlib-1.7.0",
"kotlin-stdlib-1.7.20",
"kotlin-stdlib-1.8.0",
"kotlin-stdlib-1.9.0-Beta",
"kotlin-stdlib-1.9.20-Beta",
"kotlin-stdlib-2.0.0-RC1",
)
go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
go_sdk.download(version = "1.22.2")

12
java/downgrades/BUILD.bazel Normal file
View File

@@ -0,0 +1,12 @@
load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
pkg_files(
name = "downgrades",
srcs = glob(
["**"],
exclude = ["BUILD.bazel"],
),
prefix = "downgrades",
strip_prefix = strip_prefix.from_pkg(),
visibility = ["//visibility:public"],
)

185
java/kotlin-extractor/BUILD.bazel Normal file
View File

@@ -0,0 +1,185 @@
"""
# Usage overview
Building the extractor can be done with bazel. If building from the internal repository, it is recommended to use
`tools/bazel` from there.
A specific kotlin extractor variant can be built with
```
bazel build @codeql//java/kotlin-extractor:codeql-extractor-kotlin-<variant>-<version>
```
where `<variant>` is either `standalone` or `embeddable`, and `<version>` is one of the supported versions.
```
bazel build @codeql//java/kotlin-extractor
```
will build a default variant:
* standalone, unless `CODEQL_KOTLIN_SINGLE_VERSION_EMBEDDABLE` is set to true, in which case it will go for embeddable
* the version will be taken as the last supported version less than the version of the currently installed `kotlinc`
* if `CODEQL_KOTLIN_SINGLE_VERSION` is set, that will be used instead
* if `kotlinc` is not installed, `1.9.20-Beta` will be used
If `kotlinc` is updated, bazel won't be aware of it and will therefore keep the same default version. Possible workarounds for that:
* `bazel clean`
* `bazel fetch --force @codeql//java/kotlin-extractor`
* `bazel fetch --force @codeql_kotlin_defaults//:all` (only from `codeql`)
If building from the `codeql` repository, `@codeql` can be skipped.
"""
# This file is used in the `@codeql_kotlin_embeddable` external repo, which means we need to
# reference explicitly @codeql
load(
"@codeql//java/kotlin-extractor:versions.bzl",
"VERSIONS",
"get_compatilibity_sources",
"get_language_version",
"version_less",
)
load("@rules_kotlin//kotlin:core.bzl", "kt_javac_options", "kt_kotlinc_options")
load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library")
package(default_visibility = ["//java/kotlin-extractor:__subpackages__"])
_for_embeddable = repo_name().endswith("codeql_kotlin_embeddable")
_common_extractor_name_prefix = "codeql-extractor-kotlin"
_extractor_name_prefix = "%s-%s" % (
_common_extractor_name_prefix,
"embeddable" if _for_embeddable else "standalone",
)
py_binary(
name = "generate_dbscheme",
srcs = ["generate_dbscheme.py"],
)
_resources = [
(
r,
r[len("src/main/resources/"):],
)
for r in glob(["src/main/resources/**"])
]
kt_javac_options(
name = "javac-options",
release = "8",
)
[
(
kt_kotlinc_options(
name = "kotlinc-options-%s" % v,
include_stdlibs = "none",
jvm_target = "1.8",
language_version = get_language_version(v),
warn = "error",
x_optin = [
"kotlin.RequiresOptIn",
"org.jetbrains.kotlin.ir.symbols.%s" %
("IrSymbolInternals" if version_less(v, "2.0.0") else "UnsafeDuringIrConstructionAPI"),
],
x_suppress_version_warnings = True,
),
# * extractor.name is different for each version, so we need to put it in different output dirs
# * in order to put it in `resources`, we need to define `resource_strip_prefix` to strip this version
# * `resource_strip_prefix` is unique per jar, so we must also put other resources under the same version prefix
genrule(
name = "resources-%s" % v,
srcs = [src for src, _ in _resources],
outs = [
"%s/com/github/codeql/extractor.name" % v,
] + [
"%s/%s" % (v, target)
for _, target in _resources
],
cmd = "\n".join([
"echo %s-%s > $(RULEDIR)/%s/com/github/codeql/extractor.name" % (_extractor_name_prefix, v, v),
] + [
"cp $(execpath %s) $(RULEDIR)/%s/%s" % (source, v, target)
for source, target in _resources
]),
),
kt_jvm_library(
name = "%s-%s" % (_extractor_name_prefix, v),
srcs =
["@codeql//java/kotlin-extractor:generated-dbscheme"] +
glob(
[
"src/**/*.kt",
"src/**/*.java",
],
exclude = ["src/main/kotlin/utils/versions/**"],
) + get_compatilibity_sources(v, "src/main/kotlin/utils/versions"),
javac_opts = ":javac-options",
kotlinc_opts = ":kotlinc-options-%s" % v,
module_name = "codeql-kotlin-extractor",
# resource_strip_prefix is very nit-picky: the following makes it work from
# `codeql`, `@codeql_kotlin_embeddable` and `semmle-code`
resource_strip_prefix = (
("../%s/" % repo_name() if repo_name() else "") +
("%s/" % package_name() if package_name() else "") +
v
),
resources = [
":resources-%s" % v,
],
visibility = ["//visibility:public"],
deps = [
"@kotlin-compiler%s-%s" % (
"-embeddable" if _for_embeddable else "",
v,
),
"@kotlin-stdlib-%s" % v,
],
),
# if in main repository, alias the embeddable versions from the modified @codeql_kotlin_embeddable repo
alias(
name = "%s-embeddable-%s" % (_common_extractor_name_prefix, v),
actual = "@codeql_kotlin_embeddable//:%s-embeddable-%s" % (_common_extractor_name_prefix, v),
visibility = ["//visibility:public"],
) if not _for_embeddable else None,
)
for v in VERSIONS
]
(
genrule(
name = "generated-dbscheme",
srcs = ["@codeql//java:dbscheme"],
outs = ["KotlinExtractorDbScheme.kt"],
cmd = "$(execpath :generate_dbscheme) $< $@",
tools = [":generate_dbscheme"],
visibility = ["@codeql_kotlin_embeddable//:__pkg__"],
),
[
alias(
name = n,
actual = "//java/kotlin-extractor/defaults:%s" % n,
visibility = ["//visibility:public"],
)
for n in (
"%s-standalone" % _common_extractor_name_prefix,
"%s-embeddable" % _common_extractor_name_prefix,
_common_extractor_name_prefix,
)
],
alias(
name = "kotlin-extractor",
actual = _common_extractor_name_prefix,
visibility = ["//visibility:public"],
),
filegroup(
name = "many",
srcs = ["%s-%s-%s" % (
_common_extractor_name_prefix,
variant,
version,
) for variant in ("standalone", "embeddable") for version in VERSIONS],
visibility = ["//visibility:public"],
),
sh_binary(
name = "print-default-version",
srcs = ["//java/kotlin-extractor/defaults:default-version-printer"],
),
) if not _for_embeddable else None

16
java/kotlin-extractor/current_kotlin_version.py Executable file
View File

@@ -0,0 +1,16 @@
#!/usr/bin/env python3
import subprocess
import re
import shutil
kotlinc = shutil.which('kotlinc')
if kotlinc is None:
raise Exception("kotlinc not found")
res = subprocess.run([kotlinc, "-version"], text=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
if res.returncode != 0:
raise Exception(f"kotlinc -version failed: {res.stderr}")
m = re.match(r'.* kotlinc-jvm ([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z][a-zA-Z0-9]*)?) .*', res.stderr)
if m is None:
raise Exception(f'Cannot detect version of kotlinc (got {res.stderr})')
print(m[1])

36
java/kotlin-extractor/defaults/BUILD.bazel Normal file
View File

@@ -0,0 +1,36 @@
load("@codeql_kotlin_defaults//:defaults.bzl", "kotlin_extractor_defaults")
package(default_visibility = ["//java/kotlin-extractor:__pkg__"])
_common_extractor_name_prefix = "codeql-extractor-kotlin"
alias(
name = "%s-standalone" % _common_extractor_name_prefix,
actual = "//java/kotlin-extractor:%s-standalone-%s" % (
_common_extractor_name_prefix,
kotlin_extractor_defaults.extractor_version,
),
)
alias(
name = "%s-embeddable" % _common_extractor_name_prefix,
actual = "//java/kotlin-extractor:%s-embeddable-%s" % (
_common_extractor_name_prefix,
kotlin_extractor_defaults.extractor_version,
),
)
alias(
name = _common_extractor_name_prefix,
actual = "//java/kotlin-extractor:%s-%s-%s" % (
_common_extractor_name_prefix,
kotlin_extractor_defaults.variant,
kotlin_extractor_defaults.extractor_version,
),
)
genrule(
name = "default-version-printer",
outs = ["print-default-version.sh"],
cmd = "echo echo %s > $@" % kotlin_extractor_defaults.version,
)

132
java/kotlin-extractor/deps.bzl Normal file
View File

@@ -0,0 +1,132 @@
load("//java/kotlin-extractor:versions.bzl", "VERSIONS", "version_less")
load("//misc/bazel:lfs.bzl", "lfs_smudge")
_kotlin_dep_build = """
load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_import")
kt_jvm_import(
name = "{name}",
jar = "{name}.jar",
visibility = ["//visibility:public"],
)
"""
_empty_zip = "PK\005\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
def _get_dep(repository_ctx, name):
return repository_ctx.path(Label("//java/kotlin-extractor/deps:%s" % name))
def _kotlin_dep_impl(repository_ctx):
_, _, name = repository_ctx.name.rpartition("~")
lfs_smudge(repository_ctx, [_get_dep(repository_ctx, name + ".jar")])
# for some reason rules_kotlin warns about these jars missing, this is to silence those warnings
repository_ctx.file("empty.zip", _empty_zip)
for jar in (
"annotations-13.0.jar",
"kotlin-stdlib.jar",
"kotlin-reflect.jar",
"kotlin-script-runtime.jar",
"trove4j.jar",
):
repository_ctx.symlink("empty.zip", jar)
repository_ctx.file("BUILD.bazel", _kotlin_dep_build.format(name = name))
_kotlin_dep = repository_rule(
implementation = _kotlin_dep_impl,
)
def _walk(dir):
res = []
next_dirs = [dir]
# loops must be bounded in starlark
for i in range(100):
current_dirs = next_dirs
next_dirs = []
for d in current_dirs:
children = d.readdir()
next_dirs.extend([c for c in children if c.is_dir])
res.extend([c for c in children if not c.is_dir])
if not next_dirs:
return res
fail("%s directory too deep" % dir)
def _embeddable_source_impl(repository_ctx):
src_dir = repository_ctx.path(Label("//java/kotlin-extractor:src"))
repository_ctx.watch_tree(src_dir)
for src in _walk(src_dir):
contents = repository_ctx.read(src)
contents = contents.replace(
"import com.intellij",
"import org.jetbrains.kotlin.com.intellij",
)
repository_ctx.file(str(src).replace(str(src_dir), "src"), contents)
repository_ctx.symlink(
Label("//java/kotlin-extractor:BUILD.bazel"),
"BUILD.bazel",
)
_embeddable_source = repository_rule(implementation = _embeddable_source_impl)
def _get_default_version(repository_ctx):
default_version = repository_ctx.getenv("CODEQL_KOTLIN_SINGLE_VERSION")
if default_version:
return default_version
kotlin_plugin_versions = repository_ctx.path(Label("//java/kotlin-extractor:current_kotlin_version.py"))
python = repository_ctx.which("python3") or repository_ctx.which("python")
env = {}
repository_ctx.watch(Label("//java/kotlin-extractor:dev/.kotlinc_version"))
if not repository_ctx.which("kotlinc"):
# take default from the kotlinc wrapper
path = repository_ctx.getenv("PATH")
path_to_add = repository_ctx.path(Label("//java/kotlin-extractor:dev"))
if not path:
path = str(path_to_add)
elif repository_ctx.os.name == "windows":
path = "%s;%s" % (path, path_to_add)
else:
path = "%s:%s" % (path, path_to_add)
env["PATH"] = path
res = repository_ctx.execute([python, kotlin_plugin_versions], environment = env)
if res.return_code != 0:
fail(res.stderr)
return res.stdout.strip()
def _get_available_version(version):
for available_version in reversed(VERSIONS):
if not version_less(version, available_version):
return available_version
fail("no available version found for version %s among:\n %s" % (version, " ".join(VERSIONS)))
def _defaults_impl(repository_ctx):
default_version = _get_default_version(repository_ctx)
default_variant = "standalone"
if repository_ctx.getenv("CODEQL_KOTLIN_SINGLE_VERSION_EMBEDDABLE") in ("true", "1"):
default_variant = "embeddable"
available_version = _get_available_version(default_version)
info = struct(
version = default_version,
variant = default_variant,
extractor_version = available_version,
)
repository_ctx.file(
"defaults.bzl",
"kotlin_extractor_defaults = %s\n" % repr(info),
)
repository_ctx.file("BUILD.bazel")
_defaults = repository_rule(implementation = _defaults_impl)
def _kotlin_deps_impl(module_ctx):
for v in VERSIONS:
for lib in ("compiler", "compiler-embeddable", "stdlib"):
_kotlin_dep(name = "kotlin-%s-%s" % (lib, v))
_embeddable_source(name = "codeql_kotlin_embeddable")
_defaults(name = "codeql_kotlin_defaults")
return module_ctx.extension_metadata(
root_module_direct_deps = "all",
root_module_direct_dev_deps = [],
)
kotlin_extractor_deps = module_extension(implementation = _kotlin_deps_impl)

0
java/kotlin-extractor/deps/BUILD.bazel Normal file
View File

5
java/kotlin-extractor/deps/LICENSE.md Normal file
View File

@@ -0,0 +1,5 @@
The Git LFS files contained in this directory are mirrored
from [org.jetbrains.kotlin packages in the Maven repository][1]. A copy of the license is included as
the [`license`](./license) file.
[1]: https://mvnrepository.com/artifact/org.jetbrains.kotlin

BIN
java/kotlin-extractor/deps/kotlin-compiler-1.5.0.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-1.5.10.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-1.5.20.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-1.5.30.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-1.6.0.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-1.6.20.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-1.7.0.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-1.7.20.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-1.8.0.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-1.9.0-Beta.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-1.9.20-Beta.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-2.0.0-RC1.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.0.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.10.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.20.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.30.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.0.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.20.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.20.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.8.0.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.0-Beta.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.20-Beta.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-RC1.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-stdlib-1.5.0.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-stdlib-1.5.10.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-stdlib-1.5.20.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-stdlib-1.5.30.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-stdlib-1.6.0.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-stdlib-1.6.20.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-stdlib-1.7.0.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-stdlib-1.7.20.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-stdlib-1.8.0.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-stdlib-1.9.0-Beta.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-stdlib-1.9.20-Beta.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-RC1.jar (Stored with Git LFS) Normal file
View File

Binary file not shown.

1675
java/kotlin-extractor/deps/license Normal file
View File

File diff suppressed because one or more lines are too long

2
java/kotlin-extractor/dev/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
/.kotlinc_version
/.kotlinc_installed

3
java/kotlin-extractor/dev/kotlin Executable file
View File

@@ -0,0 +1,3 @@
#!/bin/bash
exec -a "$0" "$(dirname "$0")/wrapper.py" kotlin "$@"

4
java/kotlin-extractor/dev/kotlin.bat Normal file
View File

@@ -0,0 +1,4 @@
@echo off
python "%~dp0wrapper.py" kotlin %*
exit /b %ERRORLEVEL%

3
java/kotlin-extractor/dev/kotlinc Executable file
View File

@@ -0,0 +1,3 @@
#!/bin/bash
exec -a "$0" "$(dirname "$0")/wrapper.py" kotlinc "$@"

4
java/kotlin-extractor/dev/kotlinc.bat Normal file
View File

@@ -0,0 +1,4 @@
@echo off
python "%~dp0wrapper.py" kotlinc %*
exit /b %ERRORLEVEL%

170
java/kotlin-extractor/dev/wrapper.py Executable file
View File

@@ -0,0 +1,170 @@
#!/usr/bin/env python3
"""
Wrapper script that manages kotlin versions.
Usage: add this directory to your PATH, then
* `kotlin* --select x.y.z` will select the version for the next invocations, checking it actually exists
* `kotlin* --clear` will remove any state of the wrapper (deselecting a previous version selection)
* `kotlinc -version` will print the selected version information. It will not print `JRE` information as a normal
`kotlinc` invocation would do though. In exchange, the invocation incurs no overhead.
* Any other invocation will forward to the selected kotlin tool version, downloading it if necessary. If no version was
previously selected with `--select`, a default will be used (see `DEFAULT_VERSION` below)
In order to install kotlin, ripunzip will be used if installed, or if running on Windows within `semmle-code` (ripunzip
is available in `resources/lib/windows/ripunzip` then).
"""
import pathlib
import urllib
import urllib.request
import urllib.error
import argparse
import sys
import platform
import subprocess
import zipfile
import shutil
import io
import os
DEFAULT_VERSION = "2.0.0"
def options():
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("tool")
parser.add_argument("--select")
parser.add_argument("--clear", action="store_true")
parser.add_argument("-version", action="store_true")
return parser.parse_known_args()
url_template = 'https://github.com/JetBrains/kotlin/releases/download/v{version}/kotlin-compiler-{version}.zip'
this_dir = pathlib.Path(__file__).resolve().parent
version_file = this_dir / ".kotlinc_version"
install_dir = this_dir / ".kotlinc_installed"
windows_ripunzip = this_dir.parents[4] / "resources" / "lib" / "windows" / "ripunzip" / "ripunzip.exe"
class Error(Exception):
pass
class ZipFilePreservingPermissions(zipfile.ZipFile):
def _extract_member(self, member, targetpath, pwd):
if not isinstance(member, zipfile.ZipInfo):
member = self.getinfo(member)
targetpath = super()._extract_member(member, targetpath, pwd)
attr = member.external_attr >> 16
if attr != 0:
os.chmod(targetpath, attr)
return targetpath
def check_version(version: str):
try:
with urllib.request.urlopen(url_template.format(version=version)) as response:
pass
except urllib.error.HTTPError as e:
if e.code == 404:
raise Error(f"Version {version} not found in github.com/JetBrains/kotlin/releases") from e
raise
def get_version():
try:
return version_file.read_text()
except FileNotFoundError:
return None
def install(version: str, quiet: bool):
if quiet:
info_out = subprocess.DEVNULL
info = lambda *args: None
else:
info_out = sys.stderr
info = lambda *args: print(*args, file=sys.stderr)
url = url_template.format(version=version)
if install_dir.exists():
shutil.rmtree(install_dir)
install_dir.mkdir()
ripunzip = shutil.which("ripunzip")
if ripunzip is None and platform.system() == "Windows" and windows_ripunzip.exists():
ripunzip = windows_ripunzip
if ripunzip:
info(f"downloading and extracting {url} using ripunzip")
subprocess.run([ripunzip, "unzip-uri", url], stdout=info_out, stderr=info_out, cwd=install_dir,
check=True)
return
with io.BytesIO() as buffer:
info(f"downloading {url}")
with urllib.request.urlopen(url) as response:
while True:
bytes = response.read()
if not bytes:
break
buffer.write(bytes)
buffer.seek(0)
info(f"extracting kotlin-compiler-{version}.zip")
with ZipFilePreservingPermissions(buffer) as archive:
archive.extractall(install_dir)
def forward(tool, forwarded_opts):
tool = install_dir / "kotlinc" / "bin" / tool
if platform.system() == "Windows":
tool = tool.with_suffix(".bat")
assert tool.exists(), f"{tool} not found"
args = [tool]
args.extend(forwarded_opts)
ret = subprocess.run(args).returncode
sys.exit(ret)
def clear():
if install_dir.exists():
print(f"removing {install_dir}", file=sys.stderr)
shutil.rmtree(install_dir)
if version_file.exists():
print(f"removing {version_file}", file=sys.stderr)
version_file.unlink()
def main(opts, forwarded_opts):
if opts.clear:
clear()
return
current_version = get_version()
if opts.select == "default":
selected_version = DEFAULT_VERSION
elif opts.select is not None:
check_version(opts.select)
selected_version = opts.select
else:
selected_version = current_version or DEFAULT_VERSION
if selected_version != current_version:
# don't print information about install procedure unless explicitly using --select
install(selected_version, quiet=opts.select is None)
version_file.write_text(selected_version)
if opts.select and not forwarded_opts and not opts.version:
print(f"selected {selected_version}")
return
if opts.version:
if opts.tool == "kotlinc":
print(f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)", file=sys.stderr)
return
forwarded_opts.append("-version")
forward(opts.tool, forwarded_opts)
if __name__ == "__main__":
try:
main(*options())
except Error as e:
print(f"Error: {e}", file=sys.stderr)
sys.exit(1)
except KeyboardInterrupt:
sys.exit(1)

3
java/kotlin-extractor/generate_dbscheme.py
View File

@@ -8,6 +8,7 @@ unions = {}
tables = {}
dbscheme = sys.argv[1] if len(sys.argv) >= 2 else '../ql/lib/config/semmlecode.dbscheme'
output = sys.argv[2] if len(sys.argv) >= 3 else 'src/main/kotlin/KotlinExtractorDbScheme.kt'
def parse_dbscheme(filename):
with open(filename, 'r') as f:
@@ -152,7 +153,7 @@ def genTable(kt, relname, columns, enum = None, kind = None, num = None, typ = N
kt.write(')\\n")\n')
kt.write('}\n')
with open('src/main/kotlin/KotlinExtractorDbScheme.kt', 'w') as kt:
with open(output, 'w') as kt:
kt.write('/* Generated by ' + sys.argv[0] + ': Do not edit manually. */\n')
kt.write('package com.github.codeql\n')
kt.write('import java.util.Date\n')

1
java/kotlin-extractor/kotlin_plugin_versions.py
View File

@@ -87,4 +87,3 @@ if __name__ == "__main__":
print(get_single_version(*args[2:]))
else:
raise Exception("Unknown command: " + command)

290
java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java
View File

@@ -67,7 +67,9 @@ public class OdasaOutput {
private final Logger log;
private final Compression compression;
/** DEBUG only: just use the given file as the root for TRAP, source archive etc */
/**
* DEBUG only: just use the given file as the root for TRAP, source archive etc
*/
OdasaOutput(File outputRoot, Compression compression, Logger log) {
this.trapFolder = new File(outputRoot, "trap");
this.sourceArchiveFolder = new File(outputRoot, "src_archive");
@@ -77,14 +79,16 @@ public class OdasaOutput {
}
public OdasaOutput(boolean trackClassOrigins, Compression compression, Logger log) {
String trapFolderVar = Env.systemEnv().getFirstNonEmpty("CODEQL_EXTRACTOR_JAVA_TRAP_DIR", Var.TRAP_FOLDER.name());
String trapFolderVar = Env.systemEnv().getFirstNonEmpty("CODEQL_EXTRACTOR_JAVA_TRAP_DIR",
Var.TRAP_FOLDER.name());
if (trapFolderVar == null) {
throw new ResourceError("CODEQL_EXTRACTOR_JAVA_TRAP_DIR was not set");
}
String sourceArchiveVar = Env.systemEnv().getFirstNonEmpty("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR", Var.SOURCE_ARCHIVE.name());
String sourceArchiveVar = Env.systemEnv().getFirstNonEmpty("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR",
Var.SOURCE_ARCHIVE.name());
if (sourceArchiveVar == null) {
throw new ResourceError("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR was not set");
}
}
this.trapFolder = new File(trapFolderVar);
this.sourceArchiveFolder = new File(sourceArchiveVar);
this.trackClassOrigins = trackClassOrigins;
@@ -104,6 +108,7 @@ public class OdasaOutput {
* Set the source file that is currently being processed. This may affect
* things like trap and source archive directories, and persists as a
* setting until this method is called again.
*
* @param f the current source file
*/
public void setCurrentSourceFile(File f) {
@@ -130,7 +135,7 @@ public class OdasaOutput {
private File trapSetFor(File file) {
return FileUtil.appendAbsolutePath(
currentSpecFileEntry.getTrapFolder(), PathTransformer.std().fileAsDatabaseString(file) + ".set");
currentSpecFileEntry.getTrapFolder(), PathTransformer.std().fileAsDatabaseString(file) + ".set");
}
public void addDependency(IrDeclaration sym, String signature) {
@@ -185,7 +190,8 @@ public class OdasaOutput {
return null;
return FileUtil.appendAbsolutePath(
currentSpecFileEntry.getTrapFolder(),
JARS_DIR + "/" + PathTransformer.std().fileAsDatabaseString(jarFile) + ".trap" + compression.getExtension());
JARS_DIR + "/" + PathTransformer.std().fileAsDatabaseString(jarFile) + ".trap"
+ compression.getExtension());
}
private File getTrapFileForModule(String moduleName) {
@@ -213,13 +219,13 @@ public class OdasaOutput {
private String trapFilePathForDecl(IrElement sym, String signature) {
String binaryName = getIrElementBinaryName(sym);
// TODO: Reinstate this?
//if (getTrackClassOrigins())
// classId += "-" + StringDigestor.digest(sym.getSourceFileId());
// if (getTrackClassOrigins())
// classId += "-" + StringDigestor.digest(sym.getSourceFileId());
String result = CLASSES_DIR + "/" +
binaryName.replace('.', '/') +
signature +
".members" +
".trap" + compression.getExtension();
binaryName.replace('.', '/') +
signature +
".members" +
".trap" + compression.getExtension();
return result;
}
@@ -229,16 +235,21 @@ public class OdasaOutput {
/**
* Get a {@link TrapFileManager} to write members
* about a declaration, or <code>null</code> if the declaration shouldn't be populated.
* about a declaration, or <code>null</code> if the declaration shouldn't be
* populated.
*
* @param sym
* The declaration's symbol, including, in particular, its fully qualified
* binary class name.
* The declaration's symbol, including, in particular, its
* fully qualified
* binary class name.
* @param signature
* Any unique suffix needed to distinguish `sym` from other declarations with the same name.
* For functions for example, this means its parameter signature.
* Any unique suffix needed to distinguish `sym` from other
* declarations with the same name.
* For functions for example, this means its parameter
* signature.
*/
private TrapFileManager getMembersWriterForDecl(File trap, File trapFileBase, TrapClassVersion trapFileVersion, IrElement sym, String signature) {
private TrapFileManager getMembersWriterForDecl(File trap, File trapFileBase, TrapClassVersion trapFileVersion,
IrElement sym, String signature) {
// If the TRAP file already exists then we
// don't need to write it.
if (trap.exists()) {
@@ -250,7 +261,8 @@ public class OdasaOutput {
// don't need to rewrite it only to rename it
// again.
File trapFileDir = trap.getParentFile();
File trapOld = new File(trapFileDir, trap.getName().replace(".trap" + compression.getExtension(), ".trap-old" + compression.getExtension()));
File trapOld = new File(trapFileDir,
trap.getName().replace(".trap" + compression.getExtension(), ".trap-old" + compression.getExtension()));
if (trapOld.exists()) {
log.trace("Not rewriting trap file for " + trap.toString() + " as the trap-old exists");
return null;
@@ -261,11 +273,12 @@ public class OdasaOutput {
if (trapFileBase != null && trapFileVersion != null && trapFileDir.exists()) {
String trapFileBaseName = trapFileBase.getName();
for (File f: FileUtil.list(trapFileDir)) {
for (File f : FileUtil.list(trapFileDir)) {
String name = f.getName();
Matcher m = selectClassVersionComponents.matcher(name);
if (m.matches() && m.group(1).equals(trapFileBaseName)) {
TrapClassVersion v = new TrapClassVersion(Integer.valueOf(m.group(2)), Integer.valueOf(m.group(3)), Long.valueOf(m.group(4)), m.group(5));
TrapClassVersion v = new TrapClassVersion(Integer.valueOf(m.group(2)), Integer.valueOf(m.group(3)),
Long.valueOf(m.group(4)), m.group(5));
if (v.newerThan(trapFileVersion)) {
log.trace("Not rewriting trap file for " + trap.toString() + " as " + f.toString() + " exists");
return null;
@@ -285,7 +298,8 @@ public class OdasaOutput {
return concurrentWriter(trapFile, relative, log, sym, signature);
}
private TrapFileManager concurrentWriter(File trapFile, String relative, Logger log, IrElement sym, String signature) {
private TrapFileManager concurrentWriter(File trapFile, String relative, Logger log, IrElement sym,
String signature) {
if (trapFile.exists())
return null;
return new TrapFileManager(trapFile, relative, true, log, sym, signature);
@@ -299,7 +313,8 @@ public class OdasaOutput {
private String signature;
private boolean hasError = false;
private TrapFileManager(File trapFile, String relative, boolean concurrentCreation, Logger log, IrElement sym, String signature) {
private TrapFileManager(File trapFile, String relative, boolean concurrentCreation, Logger log, IrElement sym,
String signature) {
trapDependenciesForClass = new TrapDependencies(relative);
this.trapFile = trapFile;
this.sym = sym;
@@ -325,6 +340,7 @@ public class OdasaOutput {
writeTrapDependencies(trapDependenciesForClass);
}
private void writeTrapDependencies(TrapDependencies trapDependencies) {
String dep = trapDependencies.trapFile().replace(".trap" + compression.getExtension(), ".dep");
trapDependencies.save(
@@ -340,56 +356,77 @@ public class OdasaOutput {
* Trap file locking.
*/
private final Pattern selectClassVersionComponents = Pattern.compile("(.*)#(-?[0-9]+)\\.(-?[0-9]+)-(-?[0-9]+)-(.*)\\.trap.*");
private final Pattern selectClassVersionComponents = Pattern
.compile("(.*)#(-?[0-9]+)\\.(-?[0-9]+)-(-?[0-9]+)-(.*)\\.trap.*");
/**
* <b>CAUTION</b>: to avoid the potential for deadlock between multiple concurrent extractor processes,
* only one source file {@link TrapLocker} may be open at any time, and the lock must be obtained
* <b>CAUTION</b>: to avoid the potential for deadlock between multiple
* concurrent extractor processes,
* only one source file {@link TrapLocker} may be open at any time, and the lock
* must be obtained
* <b>before</b> any <b>class</b> file lock.
*
* Trap file extensions (and paths) ensure that source and class file locks are distinct.
* Trap file extensions (and paths) ensure that source and class file locks are
* distinct.
*
* @return a {@link TrapLocker} for the currently processed source file, which must have been
* previously set by a call to {@link OdasaOutput#setCurrentSourceFile(File)}.
* @return a {@link TrapLocker} for the currently processed source file, which
* must have been
* previously set by a call to
* {@link OdasaOutput#setCurrentSourceFile(File)}.
*/
public TrapLocker getTrapLockerForCurrentSourceFile() {
return new TrapLocker((IrClass)null, null, true);
return new TrapLocker((IrClass) null, null, true);
}
/**
* <b>CAUTION</b>: to avoid the potential for deadlock between multiple concurrent extractor processes,
* only one jar file {@link TrapLocker} may be open at any time, and the lock must be obtained
* <b>after</b> any <b>source</b> file lock. Only one jar or class file lock may be open at any time.
* <b>CAUTION</b>: to avoid the potential for deadlock between multiple
* concurrent extractor processes,
* only one jar file {@link TrapLocker} may be open at any time, and the lock
* must be obtained
* <b>after</b> any <b>source</b> file lock. Only one jar or class file lock may
* be open at any time.
*
* Trap file extensions (and paths) ensure that source and jar file locks are distinct.
* Trap file extensions (and paths) ensure that source and jar file locks are
* distinct.
*
* @return a {@link TrapLocker} for the trap file corresponding to the given jar file.
* @return a {@link TrapLocker} for the trap file corresponding to the given jar
* file.
*/
public TrapLocker getTrapLockerForJarFile(File jarFile) {
return new TrapLocker(jarFile);
}
/**
* <b>CAUTION</b>: to avoid the potential for deadlock between multiple concurrent extractor processes,
* only one module {@link TrapLocker} may be open at any time, and the lock must be obtained
* <b>after</b> any <b>source</b> file lock. Only one jar or class file or module lock may be open at any time.
* <b>CAUTION</b>: to avoid the potential for deadlock between multiple
* concurrent extractor processes,
* only one module {@link TrapLocker} may be open at any time, and the lock must
* be obtained
* <b>after</b> any <b>source</b> file lock. Only one jar or class file or
* module lock may be open at any time.
*
* Trap file extensions (and paths) ensure that source and module file locks are distinct.
* Trap file extensions (and paths) ensure that source and module file locks are
* distinct.
*
* @return a {@link TrapLocker} for the trap file corresponding to the given module.
* @return a {@link TrapLocker} for the trap file corresponding to the given
* module.
*/
public TrapLocker getTrapLockerForModule(String moduleName) {
return new TrapLocker(moduleName);
}
/**
* <b>CAUTION</b>: to avoid the potential for deadlock between multiple concurrent extractor processes,
* only one class file {@link TrapLocker} may be open at any time, and the lock must be obtained
* <b>after</b> any <b>source</b> file lock. Only one jar or class file lock may be open at any time.
* <b>CAUTION</b>: to avoid the potential for deadlock between multiple
* concurrent extractor processes,
* only one class file {@link TrapLocker} may be open at any time, and the lock
* must be obtained
* <b>after</b> any <b>source</b> file lock. Only one jar or class file lock may
* be open at any time.
*
* Trap file extensions (and paths) ensure that source and class file locks are distinct.
* Trap file extensions (and paths) ensure that source and class file locks are
* distinct.
*
* @return a {@link TrapLocker} for the trap file corresponding to the given class symbol.
* @return a {@link TrapLocker} for the trap file corresponding to the given
* class symbol.
*/
public TrapLocker getTrapLockerForDecl(IrElement sym, String signature, boolean fromSource) {
return new TrapLocker(sym, signature, fromSource);
@@ -403,10 +440,11 @@ public class OdasaOutput {
private File trapFileBase = null;
private TrapClassVersion trapFileVersion = null;
private final String signature;
private TrapLocker(IrElement decl, String signature, boolean fromSource) {
this.sym = decl;
this.signature = signature;
if (sym==null) {
if (sym == null) {
log.error("Null symbol passed for Kotlin TRAP locker");
trapFile = null;
} else {
@@ -422,21 +460,25 @@ public class OdasaOutput {
// in a single directory. This makes our directory listings later slow.
// To avoid this, rather than using files named .../Foo*, we use .../Foo/Foo*.
trapFileBase = new File(new File(normalTrapFile.getParentFile(), baseName), baseName);
trapFile = new File(trapFileBase.getPath() + '#' + trapFileVersion.toString() + ".trap" + compression.getExtension());
trapFile = new File(trapFileBase.getPath() + '#' + trapFileVersion.toString() + ".trap"
+ compression.getExtension());
}
}
private TrapLocker(File jarFile) {
sym = null;
signature = null;
trapFile = getTrapFileForJarFile(jarFile);
}
private TrapLocker(String moduleName) {
sym = null;
signature = null;
trapFile = getTrapFileForModule(moduleName);
}
public TrapFileManager getTrapFileManager() {
if (trapFile!=null) {
if (trapFile != null) {
return getMembersWriterForDecl(trapFile, trapFileBase, trapFileVersion, sym, signature);
} else {
return null;
@@ -445,7 +487,7 @@ public class OdasaOutput {
@Override
public void close() {
if (trapFile!=null) {
if (trapFile != null) {
// Now that we have finished writing our TRAP file, we want
// to rename and TRAP file that matches our trapFileBase
// but doesn't have the latest metadata.
@@ -458,12 +500,13 @@ public class OdasaOutput {
String trapFileBaseName = trapFileBase.getName();
List<Pair<File, TrapClassVersion>> pairs = new LinkedList<Pair<File, TrapClassVersion>>();
for (File f: FileUtil.list(trapFileDir)) {
for (File f : FileUtil.list(trapFileDir)) {
String name = f.getName();
Matcher m = selectClassVersionComponents.matcher(name);
if (m.matches()) {
if (m.group(1).equals(trapFileBaseName)) {
TrapClassVersion v = new TrapClassVersion(Integer.valueOf(m.group(2)), Integer.valueOf(m.group(3)), Long.valueOf(m.group(4)), m.group(5));
TrapClassVersion v = new TrapClassVersion(Integer.valueOf(m.group(2)),
Integer.valueOf(m.group(3)), Long.valueOf(m.group(4)), m.group(5));
pairs.add(new Pair<File, TrapClassVersion>(f, v));
} else {
// Everything in this directory should be for the same TRAP file base
@@ -490,10 +533,12 @@ public class OdasaOutput {
};
TrapClassVersion latestVersion = Collections.max(pairs, comparator).snd();
for (Pair<File, TrapClassVersion> p: pairs) {
for (Pair<File, TrapClassVersion> p : pairs) {
if (!latestVersion.equals(p.snd())) {
File f = p.fst();
File fOld = new File(f.getParentFile(), f.getName().replace(".trap" + compression.getExtension(), ".trap-old" + compression.getExtension()));
File fOld = new File(f.getParentFile(),
f.getName().replace(".trap" + compression.getExtension(),
".trap-old" + compression.getExtension()));
// We aren't interested in whether or not this succeeds;
// it may fail because a concurrent extractor has already
// renamed it.
@@ -528,7 +573,9 @@ public class OdasaOutput {
return lastModified;
}
public String getExtractorName() { return extractorName; }
public String getExtractorName() {
return extractorName;
}
private TrapClassVersion(int majorVersion, int minorVersion, long lastModified, String extractorName) {
this.majorVersion = majorVersion;
@@ -540,24 +587,37 @@ public class OdasaOutput {
@Override
public boolean equals(Object obj) {
if (obj instanceof TrapClassVersion) {
TrapClassVersion other = (TrapClassVersion)obj;
return majorVersion == other.majorVersion && minorVersion == other.minorVersion && lastModified == other.lastModified && extractorName.equals(other.extractorName);
TrapClassVersion other = (TrapClassVersion) obj;
return majorVersion == other.majorVersion && minorVersion == other.minorVersion
&& lastModified == other.lastModified && extractorName.equals(other.extractorName);
} else {
return false;
}
}
@Override
public int hashCode() {
int hash = 7;
hash = 31 * hash + majorVersion;
hash = 31 * hash + minorVersion;
hash = 31 * hash + (int) lastModified;
hash = 31 * hash + (extractorName == null ? 0 : extractorName.hashCode());
return hash;
}
private boolean newerThan(TrapClassVersion tcv) {
// Classes being compiled from source have major version 0 but should take precedence
// Classes being compiled from source have major version 0 but should take
// precedence
// over any classes with the same qualified name loaded from the classpath
// in previous or subsequent extractor invocations.
if (tcv.majorVersion == 0 && majorVersion != 0)
return false;
else if (majorVersion == 0 && tcv.majorVersion != 0)
return true;
// Always consider the Kotlin extractor superior to the Java extractor, because we may decode and extract
// Always consider the Kotlin extractor superior to the Java extractor, because
// we may decode and extract
// Kotlin metadata that the Java extractor can't understand:
if(!Objects.equals(tcv.extractorName, extractorName)) {
if (!Objects.equals(tcv.extractorName, extractorName)) {
if (Objects.equals(tcv.extractorName, "kotlin"))
return false;
if (Objects.equals(extractorName, "kotlin"))
@@ -568,56 +628,57 @@ public class OdasaOutput {
return tcv.majorVersion < majorVersion ||
(tcv.majorVersion == majorVersion && tcv.minorVersion < minorVersion) ||
(tcv.majorVersion == majorVersion && tcv.minorVersion == minorVersion &&
tcv.lastModified < lastModified);
tcv.lastModified < lastModified);
}
private static Map<String, Map<String, Long>> jarFileEntryTimeStamps = new HashMap<>();
private static Map<String, Map<String, Long>> jarFileEntryTimeStamps = new HashMap<>();
private static Map<String, Long> getZipFileEntryTimeStamps(String path, Logger log) {
try {
Map<String, Long> result = new HashMap<>();
ZipFile zf = new ZipFile(path);
Enumeration<? extends ZipEntry> entries = zf.entries();
while (entries.hasMoreElements()) {
ZipEntry ze = entries.nextElement();
result.put(ze.getName(), ze.getLastModifiedTime().toMillis());
}
return result;
} catch(IOException e) {
log.warn("Failed to get entry timestamps from " + path, e);
return null;
}
}
private static Map<String, Long> getZipFileEntryTimeStamps(String path, Logger log) {
try {
Map<String, Long> result = new HashMap<>();
ZipFile zf = new ZipFile(path);
Enumeration<? extends ZipEntry> entries = zf.entries();
while (entries.hasMoreElements()) {
ZipEntry ze = entries.nextElement();
result.put(ze.getName(), ze.getLastModifiedTime().toMillis());
}
return result;
} catch (IOException e) {
log.warn("Failed to get entry timestamps from " + path, e);
return null;
}
}
private static long getVirtualFileTimeStamp(VirtualFile vf, Logger log) {
if (vf.getFileSystem().getProtocol().equals("jar")) {
String[] parts = vf.getPath().split("!/");
if (parts.length == 2) {
String jarFilePath = parts[0];
String entryPath = parts[1];
if (!jarFileEntryTimeStamps.containsKey(jarFilePath)) {
jarFileEntryTimeStamps.put(jarFilePath, getZipFileEntryTimeStamps(jarFilePath, log));
}
Map<String, Long> entryTimeStamps = jarFileEntryTimeStamps.get(jarFilePath);
if (entryTimeStamps != null) {
Long entryTimeStamp = entryTimeStamps.get(entryPath);
if (entryTimeStamp != null)
return entryTimeStamp;
else
log.warn("Couldn't find timestamp for jar file " + jarFilePath + " entry " + entryPath);
}
} else {
log.warn("Expected JAR-file path " + vf.getPath() + " to have exactly one '!/' separator");
}
}
private static long getVirtualFileTimeStamp(VirtualFile vf, Logger log) {
if (vf.getFileSystem().getProtocol().equals("jar")) {
String[] parts = vf.getPath().split("!/");
if (parts.length == 2) {
String jarFilePath = parts[0];
String entryPath = parts[1];
if (!jarFileEntryTimeStamps.containsKey(jarFilePath)) {
jarFileEntryTimeStamps.put(jarFilePath, getZipFileEntryTimeStamps(jarFilePath, log));
}
Map<String, Long> entryTimeStamps = jarFileEntryTimeStamps.get(jarFilePath);
if (entryTimeStamps != null) {
Long entryTimeStamp = entryTimeStamps.get(entryPath);
if (entryTimeStamp != null)
return entryTimeStamp;
else
log.warn("Couldn't find timestamp for jar file " + jarFilePath + " entry " + entryPath);
}
} else {
log.warn("Expected JAR-file path " + vf.getPath() + " to have exactly one '!/' separator");
}
}
// For all files except for jar files, and a fallback in case of I/O problems reading a jar file:
return vf.getTimeStamp();
}
// For all files except for jar files, and a fallback in case of I/O problems
// reading a jar file:
return vf.getTimeStamp();
}
private static VirtualFile getVirtualFileIfClass(IrElement e) {
if (e instanceof IrClass)
return getIrClassVirtualFile((IrClass)e);
return getIrClassVirtualFile((IrClass) e);
else
return null;
}
@@ -625,7 +686,7 @@ public class OdasaOutput {
private static TrapClassVersion fromSymbol(IrElement sym, Logger log) {
VirtualFile vf = getVirtualFileIfClass(sym);
if (vf == null && sym instanceof IrDeclaration)
vf = getVirtualFileIfClass(((IrDeclaration)sym).getParent());
vf = getVirtualFileIfClass(((IrDeclaration) sym).getParent());
if (vf == null)
return new TrapClassVersion(-1, 0, 0, null);
@@ -636,12 +697,12 @@ public class OdasaOutput {
// We want to use the latest one that there is.
Field asmField = null;
int asmNum = -1;
for(Field f : Opcodes.class.getDeclaredFields()) {
for (Field f : Opcodes.class.getDeclaredFields()) {
String name = f.getName();
if(name.startsWith("ASM")) {
if (name.startsWith("ASM")) {
try {
int i = Integer.parseInt(name.substring(3));
if(i > asmNum) {
if (i > asmNum) {
asmNum = i;
asmField = f;
}
@@ -652,26 +713,29 @@ public class OdasaOutput {
}
int asm = asmField.getInt(null);
ClassVisitor versionGetter = new ClassVisitor(asm) {
public void visit(int version, int access, java.lang.String name, java.lang.String signature, java.lang.String superName, java.lang.String[] interfaces) {
public void visit(int version, int access, java.lang.String name, java.lang.String signature,
java.lang.String superName, java.lang.String[] interfaces) {
versionStore[0] = version;
}
};
(new ClassReader(vf.contentsToByteArray())).accept(versionGetter, ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES);
(new ClassReader(vf.contentsToByteArray())).accept(versionGetter,
ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES);
return new TrapClassVersion(versionStore[0] & 0xffff, versionStore[0] >> 16, getVirtualFileTimeStamp(vf, log), "kotlin");
}
catch(IllegalAccessException e) {
return new TrapClassVersion(versionStore[0] & 0xffff, versionStore[0] >> 16,
getVirtualFileTimeStamp(vf, log), "kotlin");
} catch (IllegalAccessException e) {
log.warn("Failed to read class file version information", e);
return new TrapClassVersion(-1, 0, 0, null);
}
catch(IOException e) {
} catch (IOException e) {
log.warn("Failed to read class file version information", e);
return new TrapClassVersion(-1, 0, 0, null);
}
}
private boolean isValid() {
return majorVersion>=0 && minorVersion>=0;
return majorVersion >= 0 && minorVersion >= 0;
}
@Override
public String toString() {
return majorVersion + "." + minorVersion + "-" + lastModified + "-" + extractorName;

46
java/kotlin-extractor/versions.bzl Normal file
View File

@@ -0,0 +1,46 @@
# when updating this list, `bazel mod tidy` should be run from `codeql` to update `MODULE.bazel`
VERSIONS = [
"1.5.0",
"1.5.10",
"1.5.20",
"1.5.30",
"1.6.0",
"1.6.20",
"1.7.0",
"1.7.20",
"1.8.0",
"1.9.0-Beta",
"1.9.20-Beta",
"2.0.0-RC1",
]
def _version_to_tuple(v):
# we ignore the tag when comparing versions, for example 1.9.0-Beta <= 1.9.0
v, _, ignored_tag = v.partition("-")
return tuple([int(x) for x in v.split(".")])
def version_less(lhs, rhs):
return _version_to_tuple(lhs) < _version_to_tuple(rhs)
def get_language_version(version):
major, minor, _ = _version_to_tuple(version)
return "%s.%s" % (major, minor)
def _basename(path):
if "/" not in path:
return path
return path[path.rindex("/") + 1:]
def get_compatilibity_sources(version, dir):
prefix = "%s/v_" % dir
available = native.glob(["%s*" % prefix], exclude_directories = 0)
# we want files with the same base name to replace ones for previous versions, hence the map
srcs = {}
for d in available:
compat_version = d[len(prefix):].replace("_", ".")
if version_less(version, compat_version):
break
files = native.glob(["%s/*.kt" % d])
srcs |= {_basename(f): f for f in files}
return srcs.values()

53
java/ql/integration-tests/linux-only/kotlin/custom_plugin/build_plugin
View File

@@ -1,53 +0,0 @@
#!/usr/bin/env python3
import subprocess
import shutil
import os
import os.path
import sys
import shlex
def run_process(cmd):
try:
print("Running command: " + shlex.join(cmd))
return subprocess.run(cmd, check=True, capture_output=True)
except subprocess.CalledProcessError as e:
print("In: " + os.getcwd(), file=sys.stderr)
print("Command failed: " + shlex.join(cmd), file=sys.stderr)
print("stdout output:\n" + e.stdout.decode(encoding='UTF-8',
errors='strict'), file=sys.stderr)
print("stderr output:\n" + e.stderr.decode(encoding='UTF-8',
errors='strict'), file=sys.stderr)
raise e
root = '../../../../../../../../..'
sys.path.append(root + '/ql/java/kotlin-extractor')
import kotlin_plugin_versions
defaultKotlinDependencyVersion = kotlin_plugin_versions.get_single_version()
builddir = 'build'
dependency_dir = root + '/resources/kotlin-dependencies/'
dependencies = ['kotlin-stdlib-' + defaultKotlinDependencyVersion +
'.jar', 'kotlin-compiler-' + defaultKotlinDependencyVersion + '.jar']
classpath = ':'.join([dependency_dir + dep for dep in dependencies])
srcs = ['plugin/Plugin.kt']
output = 'plugin.jar'
if os.path.exists(builddir):
shutil.rmtree(builddir)
os.makedirs(builddir)
run_process(['kotlinc',
'-J-Xmx2G',
'-d', builddir,
'-module-name', 'test',
'-no-reflect', '-no-stdlib',
'-jvm-target', '1.8',
'-classpath', classpath] + srcs)
run_process(['jar', '-c', '-f', output,
'-C', builddir, '.',
'-C', 'plugin/resources', 'META-INF'])
shutil.rmtree(builddir)

26
java/ql/integration-tests/linux-only/kotlin/custom_plugin/plugin/BUILD.bazel Normal file
View File

@@ -0,0 +1,26 @@
load("@codeql_kotlin_defaults//:defaults.bzl", "kotlin_extractor_defaults")
load("@rules_kotlin//kotlin:core.bzl", "kt_kotlinc_options")
load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library")
load("//java/kotlin-extractor:versions.bzl", "get_language_version")
_version = kotlin_extractor_defaults.extractor_version
kt_kotlinc_options(
name = "kotlinc-options",
include_stdlibs = "none",
jvm_target = "1.8",
language_version = get_language_version(_version),
)
kt_jvm_library(
name = "plugin",
srcs = ["Plugin.kt"],
kotlinc_opts = ":kotlinc-options",
module_name = "test",
resource_strip_prefix = "%s/resources" % package_name(),
resources = glob(["resources/**"]),
deps = [
"@kotlin-compiler-%s" % _version,
"@kotlin-stdlib-%s" % _version,
],
)

24
java/ql/integration-tests/linux-only/kotlin/custom_plugin/test.py
View File

@@ -1,6 +1,26 @@
from create_database_utils import *
import subprocess
subprocess.call("./build_plugin", shell=True)
import pathlib
import shutil
this_dir = pathlib.Path(__file__).resolve().parent
cwd = pathlib.Path.cwd()
builddir = cwd / 'build'
builddir.mkdir(exist_ok=True)
try:
runSuccessfully(
[f'{get_semmle_code_path()}/tools/bazel', f'--output_user_root={builddir}', '--max_idle_secs=1', 'build',
'//java/ql/integration-tests/linux-only/kotlin/custom_plugin/plugin', '--spawn_strategy=local',
'--nouse_action_cache', '--noremote_accept_cached', '--noremote_upload_local_results',
f'--symlink_prefix={cwd / "bazel-"}'], cwd=this_dir)
finally:
# rules_python creates a read-only directory in bazel's output, this allows cleanup to succeed
runSuccessfully(['chmod', '-R', '+w', builddir])
shutil.copy(
'bazel-bin/java/ql/integration-tests/linux-only/kotlin/custom_plugin/plugin/plugin.jar', 'plugin.jar')
run_codeql_database_create(
["kotlinc -J-Xmx2G -language-version 1.9 -Xplugin=plugin.jar a.kt b.kt c.kt d.kt e.kt"], lang="java")

31
misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/MODULE.bazel Normal file
View File

@@ -0,0 +1,31 @@
module(
name = "rules_kotlin",
version = "1.9.4-codeql.1",
repo_name = "rules_kotlin",
)
bazel_dep(name = "platforms", version = "0.0.6")
bazel_dep(name = "bazel_skylib", version = "1.4.2")
bazel_dep(name = "rules_java", version = "7.2.0")
bazel_dep(name = "rules_python", version = "0.23.1")
bazel_dep(name = "rules_cc", version = "0.0.8")
rules_kotlin_extensions = use_extension(
"//src/main/starlark/core/repositories:bzlmod_setup.bzl",
"rules_kotlin_extensions",
)
use_repo(
rules_kotlin_extensions,
"com_github_google_ksp",
"com_github_jetbrains_kotlin",
"com_github_pinterest_ktlint",
"rules_android",
)
register_toolchains("//kotlin/internal:default_toolchain")
# TODO(bencodes) We should be able to remove this once rules_android has rolled out official Bzlmod support
remote_android_extensions = use_extension("@bazel_tools//tools/android:android_extensions.bzl", "remote_android_tools_extensions")
use_repo(remote_android_extensions, "android_gmaven_r8", "android_tools")
bazel_dep(name = "rules_proto", version = "5.3.0-21.7")

34
misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/codeql_add_language_version_option.patch Normal file
View File

@@ -0,0 +1,34 @@
We need to build different extractor variants with different -language-version options, which is not allowed
in current kotlin_rules
diff --git a/src/main/starlark/core/options/opts.kotlinc.bzl b/src/main/starlark/core/options/opts.kotlinc.bzl
index 9b15fb8..c0ac2cd 100644
--- a/src/main/starlark/core/options/opts.kotlinc.bzl
+++ b/src/main/starlark/core/options/opts.kotlinc.bzl
@@ -28,6 +28,11 @@ def _map_jvm_target_to_flag(version):
return None
return ["-jvm-target=%s" % version]
+def _map_language_version_to_flag(version):
+ if not version:
+ return None
+ return ["-language-version=%s" % version, "-api-version=%s" % version]
+
_KOPTS_ALL = {
"warn": struct(
args = dict(
@@ -349,6 +354,15 @@ _KOPTS_ALL = {
value_to_flag = None,
map_value_to_flag = _map_jvm_target_to_flag,
),
+ "language_version": struct(
+ args = dict(
+ default = "1.9",
+ doc = "-language-version",
+ ),
+ type = attr.string,
+ value_to_flag = None,
+ map_value_to_flag = _map_language_version_to_flag,
+ ),
}
# Filters out options that are not available in current compiler release

14
misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/codeql_do_not_emit_jdeps.patch Normal file
View File

@@ -0,0 +1,14 @@
Emitting jdeps is broken for the 2.0.0 kotlin extractor, and we don't need those files.
Patching it here rather than passing `--@rules_kotlin//kotlin/settings:jvm_emit_jdeps=false`
allows us to not have to specify that option (and therefore pull in `rules_kotlin`) in `semmle-code`.
--- a/kotlin/settings/BUILD.bazel 2000-01-01 01:00:00.000000000 +0100
+++ b/kotlin/settings/BUILD.bazel 2024-04-10 14:51:16.060085986 +0200
@@ -16,7 +16,7 @@
# Flag that controls the emission of jdeps files during kotlin jvm compilation.
bool_flag(
name = "jvm_emit_jdeps",
- build_setting_default = True, # Upstream default behavior
+ build_setting_default = False,
visibility = ["//visibility:public"],
)

9
misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/source.json Normal file
View File

@@ -0,0 +1,9 @@
{
"integrity": "sha256-dsD8wsI+33NjIK3tGs2d3guuQY5XMd8Skz2IbLqGt5U=",
"url": "https://github.com/bazelbuild/rules_kotlin/releases/download/v1.9.4/rules_kotlin-v1.9.4.tar.gz",
"patches": {
"codeql_do_not_emit_jdeps.patch": "sha256-x/HsujFlR1FGrgmbAbRZag9V4vKZZinBcs73tgRS478=",
"codeql_add_language_version_option.patch": "sha256-qFpP/hIvqGzjJi0h8LAQK0UuWqwlj/oCecZYGqlMVP8="
},
"patch_strip": 1
}

27
misc/bazel/registry/modules/rules_kotlin/metadata.json Normal file
View File

@@ -0,0 +1,27 @@
{
"homepage": "https://github.com/bazelbuild/rules_kotlin",
"maintainers": [
{
"email": "ben@ben.cm",
"github": "Bencodes",
"name": "Ben Lee"
},
{
"email": "corbin@mcneely-smith.com",
"github": "restingbull",
"name": "Corbin McNeely-Smith"
},
{
"email": "nk@snap.com",
"github": "nkoroste",
"name": "Nick Korostelev"
}
],
"repository": [
"github:bazelbuild/rules_kotlin"
],
"versions": [
"1.9.4-codeql.1"
],
"yanked_versions": {}
}