hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-25 16:55:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

Kotlin: remove obsolete scripts and reword comments

Browse Source
This commit is contained in:
Paolo Tranquilli
2024-06-11 08:52:26 +02:00
parent 317790eac3
commit 90db894d01
3 changed files with 10 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
java/kotlin-extractor/BUILD.bazel
View File

@@ -13,16 +13,20 @@ bazel build @codeql//java/kotlin-extractor
```
will build a default variant:
* standalone, unless `CODEQL_KOTLIN_SINGLE_VERSION_EMBEDDABLE` is set to true, in which case it will go for embeddable
* the version will be taken as the last supported version less than the version of the currently installed `kotlinc`
* if `CODEQL_KOTLIN_SINGLE_VERSION` is set, that will be used instead
* if `kotlinc` is not installed, `1.9.20-Beta` will be used
* the version will be taken as the last supported version less than the version of the currently available `kotlinc`,
or `CODEQL_KOTLIN_SINGLE_VERSION` if set.
If `kotlinc` is updated, bazel won't be aware of it and will therefore keep the same default version. Possible workarounds for that:
If building from the `codeql` repository, `@codeql` can be skipped.
It is recommended to use the `kotlinc` wrapper in `dev` (which is also available in `tools` from `semmle-code`), which
takes care about providing a sensible default version and keep the version of the default target up to date.
If the wrapper is not used and `kotlinc` is updated, bazel won't be aware of it and will therefore keep the same default
version. Possible workarounds for that:
* switch to using the `kotlinc` wrapper in `dev` as mentioned above
* `bazel clean`
* `bazel fetch --force @codeql//java/kotlin-extractor`
* `bazel fetch --force @codeql_kotlin_defaults//:all` (only from `codeql`)
If building from the `codeql` repository, `@codeql` can be skipped.
"""
# This file is used in the `@codeql_kotlin_embeddable` external repo, which means we need to

241
java/kotlin-extractor/build.py
View File

@@ -1,241 +0,0 @@
#!/usr/bin/env python3
import argparse
import kotlin_plugin_versions
import glob
import platform
import re
import subprocess
import shutil
import os
import os.path
import sys
import shlex
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--dependencies', default='../../../resources/kotlin-dependencies',
help='Folder containing the dependencies')
parser.add_argument('--many', action='store_true',
help='Build for all versions/kinds')
parser.add_argument('--single', action='store_false',
dest='many', help='Build for a single version/kind')
parser.add_argument('--single-version',
help='Build for a specific version/kind')
parser.add_argument('--single-version-embeddable', action='store_true',
help='When building a single version, build an embeddable extractor (default is standalone)')
return parser.parse_args()
args = parse_args()
def is_windows():
'''Whether we appear to be running on Windows'''
if platform.system() == 'Windows':
return True
if platform.system().startswith('CYGWIN'):
return True
return False
# kotlinc might be kotlinc.bat or kotlinc.cmd on Windows, so we use `which` to find out what it is
kotlinc = shutil.which('kotlinc')
if kotlinc is None:
print("Cannot build the Kotlin extractor: no kotlinc found on your PATH", file=sys.stderr)
sys.exit(1)
javac = 'javac'
kotlin_dependency_folder = args.dependencies
def quote_for_batch(arg):
if ';' in arg or '=' in arg:
if '"' in arg:
raise Exception('Need to quote something containing a quote')
return '"' + arg + '"'
else:
return arg
def run_process(cmd, capture_output=False):
print("Running command: " + shlex.join(cmd))
if is_windows():
cmd = ' '.join(map(quote_for_batch, cmd))
print("Converted to Windows command: " + cmd)
try:
if capture_output:
return subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
else:
return subprocess.run(cmd, check=True)
except subprocess.CalledProcessError as e:
print("In: " + os.getcwd(), file=sys.stderr)
shell_cmd = cmd if is_windows() else shlex.join(cmd)
print("Command failed: " + shell_cmd, file=sys.stderr)
if capture_output:
print("stdout output:\n" + e.stdout.decode(encoding='UTF-8',
errors='replace'), file=sys.stderr)
print("stderr output:\n" + e.stderr.decode(encoding='UTF-8',
errors='replace'), file=sys.stderr)
raise e
def write_arg_file(arg_file, args):
with open(arg_file, 'w') as f:
for arg in args:
if "'" in arg:
raise Exception('Single quote in argument: ' + arg)
f.write("'" + arg.replace('\\', '/') + "'\n")
def compile_to_dir(build_dir, srcs, version, classpath, java_classpath, output):
# Use kotlinc to compile .kt files:
kotlin_arg_file = build_dir + '/kotlin.args'
opt_in_args = ['-opt-in=kotlin.RequiresOptIn']
if version.lessThan(kotlin_plugin_versions.Version(2, 0, 0, "")):
opt_in_args.append('-opt-in=org.jetbrains.kotlin.ir.symbols.IrSymbolInternals')
else:
opt_in_args.append('-opt-in=org.jetbrains.kotlin.ir.symbols.UnsafeDuringIrConstructionAPI')
kotlin_args = ['-Werror'] \
+ opt_in_args \
+ ['-d', output,
'-module-name', 'codeql-kotlin-extractor',
'-Xsuppress-version-warnings',
'-language-version', version.toLanguageVersionString(),
'-no-reflect', '-no-stdlib',
'-jvm-target', '1.8',
'-classpath', classpath] + srcs
write_arg_file(kotlin_arg_file, kotlin_args)
run_process([kotlinc,
# kotlinc can default to 256M, which isn't enough when we are extracting the build
'-J-Xmx2G',
'@' + kotlin_arg_file])
# Use javac to compile .java files, referencing the Kotlin class files:
java_arg_file = build_dir + '/java.args'
java_args = ['-d', output,
'-source', '8', '-target', '8',
'-classpath', os.path.pathsep.join([output, classpath, java_classpath])] \
+ [s for s in srcs if s.endswith(".java")]
write_arg_file(java_arg_file, java_args)
run_process([javac, '@' + java_arg_file])
def compile_to_jar(build_dir, tmp_src_dir, srcs, version, classpath, java_classpath, output):
class_dir = build_dir + '/classes'
if os.path.exists(class_dir):
shutil.rmtree(class_dir)
os.makedirs(class_dir)
compile_to_dir(build_dir, srcs, version, classpath, java_classpath, class_dir)
run_process(['jar', 'cf', output,
'-C', class_dir, '.',
'-C', tmp_src_dir + '/main/resources', 'META-INF',
'-C', tmp_src_dir + '/main/resources', 'com/github/codeql/extractor.name'])
shutil.rmtree(class_dir)
def find_sources(path):
return glob.glob(path + '/**/*.kt', recursive=True) + glob.glob(path + '/**/*.java', recursive=True)
def find_jar(path, base):
fn = path + '/' + base + '.jar'
if not os.path.isfile(fn):
raise Exception('Cannot find jar file at %s' % fn)
return fn
def bases_to_classpath(path, bases):
result = []
for base in bases:
result.append(find_jar(path, base))
return os.path.pathsep.join(result)
def transform_to_embeddable(srcs):
# replace imports in files:
for src in srcs:
with open(src, 'r') as f:
content = f.read()
content = content.replace('import com.intellij',
'import org.jetbrains.kotlin.com.intellij')
with open(src, 'w') as f:
f.write(content)
def compile(jars, java_jars, dependency_folder, transform_to_embeddable, output, build_dir, version_str):
classpath = bases_to_classpath(dependency_folder, jars)
java_classpath = bases_to_classpath(dependency_folder, java_jars)
tmp_src_dir = build_dir + '/temp_src'
if os.path.exists(tmp_src_dir):
shutil.rmtree(tmp_src_dir)
shutil.copytree('src', tmp_src_dir)
include_version_folder = tmp_src_dir + '/main/kotlin/utils/this_version'
os.makedirs(include_version_folder)
resource_dir = tmp_src_dir + '/main/resources/com/github/codeql'
os.makedirs(resource_dir)
with open(resource_dir + '/extractor.name', 'w') as f:
f.write(output)
version = kotlin_plugin_versions.version_string_to_version(version_str)
for a_version in kotlin_plugin_versions.many_versions_versions_asc:
if a_version.lessThanOrEqual(version):
d = tmp_src_dir + '/main/kotlin/utils/versions/v_' + \
a_version.toString().replace('.', '_')
if os.path.exists(d):
# copy and overwrite files from the version folder to the include folder
shutil.copytree(d, include_version_folder, dirs_exist_ok=True)
# remove all version folders:
shutil.rmtree(tmp_src_dir + '/main/kotlin/utils/versions')
srcs = find_sources(tmp_src_dir)
transform_to_embeddable(srcs)
compile_to_jar(build_dir, tmp_src_dir, srcs, version, classpath, java_classpath, output)
shutil.rmtree(tmp_src_dir)
def compile_embeddable(version):
compile(['kotlin-stdlib-' + version, 'kotlin-compiler-embeddable-' + version],
['kotlin-stdlib-' + version],
kotlin_dependency_folder,
transform_to_embeddable,
'codeql-extractor-kotlin-embeddable-%s.jar' % (version),
'build_embeddable_' + version,
version)
def compile_standalone(version):
compile(['kotlin-stdlib-' + version, 'kotlin-compiler-' + version],
['kotlin-stdlib-' + version],
kotlin_dependency_folder,
lambda srcs: None,
'codeql-extractor-kotlin-standalone-%s.jar' % (version),
'build_standalone_' + version,
version)
if args.single_version:
if args.single_version_embeddable == True:
compile_embeddable(args.single_version)
else:
compile_standalone(args.single_version)
elif args.single_version_embeddable == True:
print("--single-version-embeddable requires --single-version", file=sys.stderr)
sys.exit(1)
elif args.many:
for version in kotlin_plugin_versions.many_versions:
compile_standalone(version)
compile_embeddable(version)
else:
compile_standalone(kotlin_plugin_versions.get_single_version())

89
java/kotlin-extractor/kotlin_plugin_versions.py
View File

@@ -1,89 +0,0 @@
#!/usr/bin/python
import platform
import re
import shutil
import subprocess
import sys
def is_windows():
'''Whether we appear to be running on Windows'''
if platform.system() == 'Windows':
return True
if platform.system().startswith('CYGWIN'):
return True
return False
class Version:
def __init__(self, major, minor, patch, tag):
self.major = major
self.minor = minor
self.patch = patch
self.tag = tag
def toTupleWithTag(self):
return [self.major, self.minor, self.patch, self.tag]
def toTupleNoTag(self):
return [self.major, self.minor, self.patch]
def lessThan(self, other):
return self.toTupleNoTag() < other.toTupleNoTag()
def lessThanOrEqual(self, other):
return self.toTupleNoTag() <= other.toTupleNoTag()
def toString(self):
return f'{self.major}.{self.minor}.{self.patch}{self.tag}'
def toLanguageVersionString(self):
return f'{self.major}.{self.minor}'
def version_string_to_version(version):
m = re.match(r'([0-9]+)\.([0-9]+)\.([0-9]+)(.*)', version)
return Version(int(m.group(1)), int(m.group(2)), int(m.group(3)), m.group(4))
# Version number used by CI.
ci_version = '2.0.0'
many_versions = [ '1.5.0', '1.5.10', '1.5.20', '1.5.30', '1.6.0', '1.6.20', '1.7.0', '1.7.20', '1.8.0', '1.9.0-Beta', '1.9.20-Beta', '2.0.0-RC1' ]
many_versions_versions = [version_string_to_version(v) for v in many_versions]
many_versions_versions_asc = sorted(many_versions_versions, key = lambda v: v.toTupleWithTag())
many_versions_versions_desc = reversed(many_versions_versions_asc)
class KotlincNotFoundException(Exception):
pass
def get_single_version(fakeVersionOutput = None):
# kotlinc might be kotlinc.bat or kotlinc.cmd on Windows, so we use `which` to find out what it is
kotlinc = shutil.which('kotlinc')
if kotlinc is None:
raise KotlincNotFoundException()
versionOutput = subprocess.run([kotlinc, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).stderr if fakeVersionOutput is None else fakeVersionOutput
m = re.match(r'.* kotlinc-jvm ([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z][a-zA-Z0-9]*)?) .*', versionOutput)
if m is None:
raise Exception('Cannot detect version of kotlinc (got ' + str(versionOutput) + ')')
current_version = version_string_to_version(m.group(1))
for version in many_versions_versions_desc:
if version.lessThanOrEqual(current_version):
return version.toString()
raise Exception(f'No suitable kotlinc version found for {current_version} (got {versionOutput}; know about {str(many_versions)})')
def get_latest_url():
url = 'https://github.com/JetBrains/kotlin/releases/download/v' + ci_version + '/kotlin-compiler-' + ci_version + '.zip'
return url
if __name__ == "__main__":
args = sys.argv
if len(args) < 2:
raise Exception("Bad arguments")
command = args[1]
if command == 'latest-url':
print(get_latest_url())
elif command == 'single-version':
print(get_single_version(*args[2:]))
else:
raise Exception("Unknown command: " + command)