From b07fa70133168666307419d44bc52348ca2f84d6 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 15 Apr 2024 10:48:01 +0200 Subject: [PATCH 1/3] Kotlin/Bazel: provide wrapper for managing versions of `kotlinc` By adding `java/kotlinc-extractor/deps/dev` to `PATH`, one gets a `kotlinc` wrapper that takes care of downloading and extracting the desired version of `kotlinc` on demand. The desired version can be selected with `kotlinc --select x.y.z`, or left to the current default of `1.9.0`. Moreover, this default version is integrated with the Bazel build, so that when using this wrapper, changes in the selected version will be picked up to define the default single version kotlin extractor build, without needing to do anything else (like `bazel fetch --force` or similar). Selected and installed version data is stored in `.gitignore`d files in the same directory, and can be cleared with `kotlinc --clear`. --- java/kotlin-extractor/deps.bzl | 19 ++- java/kotlin-extractor/deps/dev/.gitignore | 3 + java/kotlin-extractor/deps/dev/kotlinc | 156 +++++++++++++++++++++ java/kotlin-extractor/deps/dev/kotlinc.bat | 2 + java/kotlin-extractor/versions.bzl | 2 - 5 files changed, 176 insertions(+), 6 deletions(-) create mode 100644 java/kotlin-extractor/deps/dev/.gitignore create mode 100755 java/kotlin-extractor/deps/dev/kotlinc create mode 100644 java/kotlin-extractor/deps/dev/kotlinc.bat diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 4fb1d0b38af..8d737211967 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -1,4 +1,4 @@ -load("//java/kotlin-extractor:versions.bzl", "DEFAULT_VERSION", "VERSIONS", "version_less") +load("//java/kotlin-extractor:versions.bzl", "VERSIONS", "version_less") load("//misc/bazel:lfs.bzl", "lfs_smudge") _kotlin_dep_build = """ @@ -73,11 +73,22 @@ def _get_default_version(repository_ctx): default_version = repository_ctx.getenv("CODEQL_KOTLIN_SINGLE_VERSION") if default_version: return default_version - if not repository_ctx.which("kotlinc"): - return DEFAULT_VERSION kotlin_plugin_versions = repository_ctx.path(Label("//java/kotlin-extractor:current_kotlin_version.py")) python = repository_ctx.which("python3") or repository_ctx.which("python") - res = repository_ctx.execute([python, kotlin_plugin_versions]) + env = {} + repository_ctx.watch(Label("//java/kotlin-extractor/deps:dev/.kotlinc_selected_version")) + if not repository_ctx.which("kotlinc"): + # take default from the kotlinc wrapper + path = repository_ctx.getenv("PATH") + path_to_add = repository_ctx.path(Label("//java/kotlin-extractor/deps:dev")) + if not path: + path = str(path_to_add) + elif repository_ctx.os.name == "windows": + path = "%s;%s" % (path, path_to_add) + else: + path = "%s:%s" % (path, path_to_add) + env["PATH"] = path + res = repository_ctx.execute([python, kotlin_plugin_versions], environment = env) if res.return_code != 0: fail(res.stderr) return res.stdout.strip() diff --git a/java/kotlin-extractor/deps/dev/.gitignore b/java/kotlin-extractor/deps/dev/.gitignore new file mode 100644 index 00000000000..a65f75fad6f --- /dev/null +++ b/java/kotlin-extractor/deps/dev/.gitignore @@ -0,0 +1,3 @@ +/.kotlinc_installed +/.kotlinc_installed_version +/.kotlinc_selected_version diff --git a/java/kotlin-extractor/deps/dev/kotlinc b/java/kotlin-extractor/deps/dev/kotlinc new file mode 100755 index 00000000000..4f23c5e1d3d --- /dev/null +++ b/java/kotlin-extractor/deps/dev/kotlinc @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 + +""" +Wrapper script that manages kotlinc versions. +Usage: add this directory to your PATH, then +* `kotlinc --select x.y.z` will select the version for the next invocations, checking it actually exists +* `kotlinc --clear` will remove any state of the wrapper (deselecting a previous version selection) +* `kotlinc -version` will print the selected version information. It will not print `JRE` information as a normal + `kotlinc` invocation would do though. In exchange, the invocation incurs no overhead. +* Any other invocation will forward to the selected kotlinc version, downloading it if necessary. If no version was + previously selected with `--select`, a default will be used (see `DEFAULT_VERSION` below) + +In order to install kotlin, ripunzip will be used if installed, or if running on Windows within `semmle-code` (ripunzip +is available in `resources/lib/windows/ripunzip` then). +""" + +import pathlib +import os +import urllib +import urllib.request +import urllib.error +import argparse +import sys +import platform +import subprocess +import zipfile +import shutil +import io + +DEFAULT_VERSION = "1.9.0" + +def options(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--select") + parser.add_argument("--clear", action="store_true") + parser.add_argument("-version", action="store_true") + return parser.parse_known_args() + + +url_template = 'https://github.com/JetBrains/kotlin/releases/download/v{version}/kotlin-compiler-{version}.zip' +this_dir = pathlib.Path(__file__).resolve().parent +version_file = this_dir / ".kotlinc_selected_version" +installed_version_file = this_dir / ".kotlinc_installed_version" +install_dir = this_dir / ".kotlinc_installed" +windows_ripunzip = this_dir.parents[4] / "resources" / "lib" / "windows" / "ripunzip" / "ripunzip.exe" + + +class Error(Exception): + pass + + +class ZipFilePreservingPermissions(zipfile.ZipFile): + def _extract_member(self, member, targetpath, pwd): + if not isinstance(member, zipfile.ZipInfo): + member = self.getinfo(member) + + targetpath = super()._extract_member(member, targetpath, pwd) + + attr = member.external_attr >> 16 + if attr != 0: + os.chmod(targetpath, attr) + return targetpath + + +def check_version(version: str): + try: + with urllib.request.urlopen(url_template.format(version=version)) as response: + pass + except urllib.error.HTTPError as e: + if e.code == 404: + raise Error(f"Version {version} not found in github.com/JetBrains/kotlin/releases") from e + raise + +def get_version(file: pathlib.Path) -> str: + try: + return file.read_text() + except FileNotFoundError: + return None + + +def install(version: str): + url = url_template.format(version=version) + if install_dir.exists(): + shutil.rmtree(install_dir) + install_dir.mkdir() + ripunzip = shutil.which("ripunzip") + if platform.system() == "Windows" and windows_ripunzip.exists(): + ripunzip = windows_ripunzip + if ripunzip: + print(f"downloading and extracting {url} using ripunzip", file=sys.stderr) + subprocess.run([ripunzip, "unzip-uri", url], cwd=install_dir, check=True) + return + with io.BytesIO() as buffer: + with urllib.request.urlopen(url) as response: + print(f"downloading {url}", file=sys.stderr) + while True: + bytes = response.read() + if not bytes: + break + buffer.write(bytes) + buffer.seek(0) + print(f"extracting kotlin-compiler-{version}.zip", file=sys.stderr) + with ZipFilePreservingPermissions(buffer) as archive: + archive.extractall(install_dir) + + +def forward(forwarded_opts): + kotlinc = install_dir / "kotlinc" / "bin" / "kotlinc" + if platform.system() == "Windows": + kotlinc = kotlinc.with_suffix(".bat") + args = [sys.argv[0]] + args.extend(forwarded_opts) + os.execv(kotlinc, args) + +def clear(): + if install_dir.exists(): + print(f"removing {install_dir}", file=sys.stderr) + shutil.rmtree(install_dir) + if installed_version_file.exists(): + print(f"removing {installed_version_file}", file=sys.stderr) + installed_version_file.unlink() + if version_file.exists(): + print(f"removing {version_file}", file=sys.stderr) + version_file.unlink() + +def main(opts, forwarded_opts): + if opts.clear: + clear() + return + if opts.select: + check_version(opts.select) + version_file.write_text(opts.select) + selected_version = opts.select + else: + selected_version = get_version(version_file) + if not selected_version: + selected_version = DEFAULT_VERSION + version_file.write_text(selected_version) + if opts.version: + print(f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)", file=sys.stderr) + return + if opts.select and not forwarded_opts: + print(f"selected {selected_version}", file=sys.stderr) + return + if get_version(installed_version_file) != selected_version: + install(selected_version) + installed_version_file.write_text(selected_version) + forward(forwarded_opts) + + +if __name__ == "__main__": + try: + main(*options()) + except Exception as e: + print(f"{e.__class__.__name__}: {e}", file=sys.stderr) + sys.exit(1) diff --git a/java/kotlin-extractor/deps/dev/kotlinc.bat b/java/kotlin-extractor/deps/dev/kotlinc.bat new file mode 100644 index 00000000000..3d75e939610 --- /dev/null +++ b/java/kotlin-extractor/deps/dev/kotlinc.bat @@ -0,0 +1,2 @@ +@echo off +python3 %~dp0/kotlinc diff --git a/java/kotlin-extractor/versions.bzl b/java/kotlin-extractor/versions.bzl index 827ea9adf3b..728672ead6d 100644 --- a/java/kotlin-extractor/versions.bzl +++ b/java/kotlin-extractor/versions.bzl @@ -14,8 +14,6 @@ VERSIONS = [ "2.0.0-RC1", ] -DEFAULT_VERSION = "1.9.0" - def _version_to_tuple(v): # we ignore the tag when comparing versions, for example 1.9.0-Beta <= 1.9.0 v, _, ignored_tag = v.partition("-") From e53ef4acd2f9f216c2266f32d8b25686cffc8ff2 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 15 Apr 2024 13:13:31 +0200 Subject: [PATCH 2/3] Kotlin/Bazel: fix wrapper on Windows --- java/kotlin-extractor/deps/dev/kotlinc | 11 ++++++++--- java/kotlin-extractor/deps/dev/kotlinc.bat | 4 +++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/java/kotlin-extractor/deps/dev/kotlinc b/java/kotlin-extractor/deps/dev/kotlinc index 4f23c5e1d3d..f74ed42c509 100755 --- a/java/kotlin-extractor/deps/dev/kotlinc +++ b/java/kotlin-extractor/deps/dev/kotlinc @@ -15,7 +15,6 @@ is available in `resources/lib/windows/ripunzip` then). """ import pathlib -import os import urllib import urllib.request import urllib.error @@ -29,6 +28,7 @@ import io DEFAULT_VERSION = "1.9.0" + def options(): parser = argparse.ArgumentParser(add_help=False) parser.add_argument("--select") @@ -71,6 +71,7 @@ def check_version(version: str): raise Error(f"Version {version} not found in github.com/JetBrains/kotlin/releases") from e raise + def get_version(file: pathlib.Path) -> str: try: return file.read_text() @@ -108,9 +109,12 @@ def forward(forwarded_opts): kotlinc = install_dir / "kotlinc" / "bin" / "kotlinc" if platform.system() == "Windows": kotlinc = kotlinc.with_suffix(".bat") - args = [sys.argv[0]] + assert kotlinc.exists(), f"{kotlinc} not found" + args = [kotlinc] args.extend(forwarded_opts) - os.execv(kotlinc, args) + ret = subprocess.run(args).returncode + sys.exit(ret) + def clear(): if install_dir.exists(): @@ -123,6 +127,7 @@ def clear(): print(f"removing {version_file}", file=sys.stderr) version_file.unlink() + def main(opts, forwarded_opts): if opts.clear: clear() diff --git a/java/kotlin-extractor/deps/dev/kotlinc.bat b/java/kotlin-extractor/deps/dev/kotlinc.bat index 3d75e939610..f6e59843f76 100644 --- a/java/kotlin-extractor/deps/dev/kotlinc.bat +++ b/java/kotlin-extractor/deps/dev/kotlinc.bat @@ -1,2 +1,4 @@ @echo off -python3 %~dp0/kotlinc + +python "%~dp0/kotlinc" %* +exit /b %ERRORLEVEL% From 1b5675eb214a20d332e39d9f8d34b1655c0bd9b1 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 15 Apr 2024 13:31:29 +0200 Subject: [PATCH 3/3] Kotlin/Bazel: tweak wrapper --- java/kotlin-extractor/deps/dev/kotlinc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/java/kotlin-extractor/deps/dev/kotlinc b/java/kotlin-extractor/deps/dev/kotlinc index f74ed42c509..509ed30cb7c 100755 --- a/java/kotlin-extractor/deps/dev/kotlinc +++ b/java/kotlin-extractor/deps/dev/kotlinc @@ -25,6 +25,7 @@ import subprocess import zipfile import shutil import io +import os DEFAULT_VERSION = "1.9.0" @@ -85,15 +86,15 @@ def install(version: str): shutil.rmtree(install_dir) install_dir.mkdir() ripunzip = shutil.which("ripunzip") - if platform.system() == "Windows" and windows_ripunzip.exists(): + if ripunzip is None and platform.system() == "Windows" and windows_ripunzip.exists(): ripunzip = windows_ripunzip if ripunzip: print(f"downloading and extracting {url} using ripunzip", file=sys.stderr) subprocess.run([ripunzip, "unzip-uri", url], cwd=install_dir, check=True) return with io.BytesIO() as buffer: + print(f"downloading {url}", file=sys.stderr) with urllib.request.urlopen(url) as response: - print(f"downloading {url}", file=sys.stderr) while True: bytes = response.read() if not bytes: @@ -141,15 +142,12 @@ def main(opts, forwarded_opts): if not selected_version: selected_version = DEFAULT_VERSION version_file.write_text(selected_version) - if opts.version: - print(f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)", file=sys.stderr) - return - if opts.select and not forwarded_opts: - print(f"selected {selected_version}", file=sys.stderr) - return if get_version(installed_version_file) != selected_version: install(selected_version) installed_version_file.write_text(selected_version) + if opts.version or (opts.select and not forwarded_opts): + print(f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)", file=sys.stderr) + return forward(forwarded_opts) @@ -159,3 +157,5 @@ if __name__ == "__main__": except Exception as e: print(f"{e.__class__.__name__}: {e}", file=sys.stderr) sys.exit(1) + except KeyboardInterrupt: + sys.exit(1)