Merge pull request #18215 from smowton/smowton/ke2/basic-generics

KE2: Upgrade to Kotlin 2.1.0; restore basic type parameter and type argument extraction

.gitattributes

@@ -50,6 +50,7 @@
 *.dll -text
 *.pdb -text
 
+/maven_install.json linguist-generated=true
 /java/ql/test/stubs/**/*.java linguist-generated=true
 /java/ql/test/experimental/stubs/**/*.java linguist-generated=true
 /java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text

.github/labeler.yml

@@ -11,7 +11,7 @@ Go:
   - change-notes/**/*go.*
 
 Java:
-  - any: [ 'java/**/*', '!java/kotlin-extractor/**/*', '!java/ql/test/kotlin/**/*' ]
+  - any: [ 'java/**/*', '!java/kotlin-extractor/**/*', '!java/kotlin-extractor2/**/*', '!java/ql/test-kotlin*/**/*' ]
   - change-notes/**/*java.*
 
 JS:
@@ -20,6 +20,7 @@ JS:
 
 Kotlin:
   - java/kotlin-extractor/**/*
+  - java/kotlin-extractor2/**/*
   - java/ql/test-kotlin*/**/*
 
 Python:

CODEOWNERS

@@ -10,6 +10,7 @@
 /swift/ @github/codeql-swift
 /misc/codegen/ @github/codeql-swift
 /java/kotlin-extractor/ @github/codeql-kotlin
+/java/kotlin-extractor2/ @github/codeql-kotlin
 /java/ql/test-kotlin1/ @github/codeql-kotlin
 /java/ql/test-kotlin2/ @github/codeql-kotlin
 

MODULE.bazel

@@ -28,6 +28,7 @@ bazel_dep(name = "gazelle", version = "0.38.0")
 bazel_dep(name = "rules_dotnet", version = "0.17.4")
 bazel_dep(name = "googletest", version = "1.14.0.bcr.1")
 bazel_dep(name = "rules_rust", version = "0.52.2")
+bazel_dep(name = "rules_jvm_external", version = "6.2")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
@@ -162,6 +163,35 @@ use_repo(
     "kotlin-stdlib-2.1.0-Beta1",
 )
 
+maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven")
+
+# run
+#   REPIN=1 bazel run @maven_deps//:pin
+# from this directory after modifying the following to update maven_install.json
+maven.install(
+    name = "maven_deps",
+    # The Caffeine version needs to match https://github.com/JetBrains/kotlin/blob/master/gradle/libs.versions.toml
+    # See also https://youtrack.jetbrains.com/issue/KT-73751/Analysis-API-Caffeine-dependency which seeks a better
+    # way of including the needed dependency.
+    artifacts = [
+        "org.jetbrains.kotlin:%s:2.1.0" % kotlin_lib
+        for kotlin_lib in ("kotlin-annotation-processing", "kotlin-compiler")
+    ] + [ "com.github.ben-manes.caffeine:caffeine:2.9.3" ]  ,
+    lock_file = "//:maven_install.json",
+    repositories = [
+        "https://repo1.maven.org/maven2",
+        # some of these URLs might be needed at some point
+        #        "https://maven.pkg.jetbrains.space/kotlin/p/kotlin/bootstrap",
+        #        "https://maven.pkg.jetbrains.space/kotlin/p/kotlin/kotlin-ide-plugin-dependencies",
+        #        "https://www.jetbrains.com/intellij-repository/releases",
+        #        "https://cache-redirector.jetbrains.com/intellij-third-party-dependencies",
+    ],
+)
+use_repo(
+    maven,
+    "maven_deps",
+)
+
 go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
 go_sdk.download(version = "1.23.1")
 

java/kotlin-extractor/BUILD.bazel

@@ -38,6 +38,7 @@ load(
     "get_language_version",
     "version_less",
 )
+load("@codeql//misc/bazel:pkg.bzl", "codeql_pkg_files")
 load("@rules_kotlin//kotlin:core.bzl", "kt_javac_options", "kt_kotlinc_options")
 load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library")
 
@@ -173,9 +174,9 @@ kt_javac_options(
             _common_extractor_name_prefix,
         )
     ],
-    alias(
+    codeql_pkg_files(
         name = "kotlin-extractor",
-        actual = _common_extractor_name_prefix,
+        srcs = [_common_extractor_name_prefix],
         visibility = ["//visibility:public"],
     ),
     filegroup(

java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt

@@ -2776,7 +2776,7 @@ open class KotlinFileExtractor(
             val locId = tw.getLocation(ta)
             // TODO: We don't really want to generate any Java types here; we only want the KT type:
             val type = useType(ta.expandedType)
-            tw.writeKt_type_alias(id, ta.name.asString(), type.kotlinResult.id)
+            TODO() // TODO: KotType tw.writeKt_type_alias(id, ta.name.asString(), type.kotlinResult.id)
             tw.writeHasLocation(id, locId)
 
             // TODO: extract annotations
@@ -6274,7 +6274,12 @@ open class KotlinFileExtractor(
                         val bLocId = tw.getLocation(b)
                         tw.writeStmts_whenbranch(bId, id, i, callable)
                         tw.writeHasLocation(bId, bLocId)
-                        extractExpressionExpr(b.condition, callable, bId, 0, bId)
+
+                        val condId = tw.getFreshIdLabel<DbWhenbranchcondition>()
+                        tw.writeStmts_whenbranchcondition(condId, bId, 0, callable)
+                        tw.writeHasLocation(condId, bLocId)
+                        tw.writeWhen_branch_condition_with_expr(condId)
+                        extractExpressionExpr(b.condition, callable, condId, 0, condId)
                         extractExpressionStmt(b.result, callable, bId, 1)
                         if (b is IrElseBranch) {
                             tw.writeWhen_branch_else(bId)
@@ -7569,11 +7574,11 @@ open class KotlinFileExtractor(
             val locId = tw.getLocation(propertyReferenceExpr)
 
             val javaResult = TypeResult(tw.getFreshIdLabel<DbClassorinterface>(), "", "")
-            val kotlinResult = TypeResult(tw.getFreshIdLabel<DbKt_notnull_type>(), "", "")
-            tw.writeKt_notnull_types(kotlinResult.id, javaResult.id)
+            TODO() // TODO:KotType val kotlinResult = TypeResult(tw.getFreshIdLabel<DbKt_notnull_type>(), "", "")
+            TODO() // TODO:KotType tw.writeKt_notnull_types(kotlinResult.id, javaResult.id)
             val ids =
                 GeneratedClassLabels(
-                    TypeResults(javaResult, kotlinResult),
+                    TODO(), // TODO:KotType TypeResults(javaResult, kotlinResult),
                     constructor = tw.getFreshIdLabel(),
                     constructorBlock = tw.getFreshIdLabel()
                 )
@@ -7669,6 +7674,8 @@ open class KotlinFileExtractor(
                         classId,
                         locId
                     )
+                TODO() // TODO:KotType
+                /*
                 val fieldId = useField(backingField.owner)
 
                 helper.extractFieldReturnOfReflectionTarget(getLabels, backingField)
@@ -7680,6 +7687,7 @@ open class KotlinFileExtractor(
                     getterParameterTypes,
                     getterReturnType
                 )
+                */
             }
 
             if (setter != null) {
@@ -7890,11 +7898,11 @@ open class KotlinFileExtractor(
             val locId = tw.getLocation(functionReferenceExpr)
 
             val javaResult = TypeResult(tw.getFreshIdLabel<DbClassorinterface>(), "", "")
-            val kotlinResult = TypeResult(tw.getFreshIdLabel<DbKt_notnull_type>(), "", "")
-            tw.writeKt_notnull_types(kotlinResult.id, javaResult.id)
+            TODO() // TODO:KotType val kotlinResult = TypeResult(tw.getFreshIdLabel<DbKt_notnull_type>(), "", "")
+            TODO() // TODO:KotType tw.writeKt_notnull_types(kotlinResult.id, javaResult.id)
             val ids =
                 LocallyVisibleFunctionLabels(
-                    TypeResults(javaResult, kotlinResult),
+                    TODO(), // TODO:KotType TypeResults(javaResult, kotlinResult),
                     constructor = tw.getFreshIdLabel(),
                     function = tw.getFreshIdLabel(),
                     constructorBlock = tw.getFreshIdLabel()
@@ -8782,11 +8790,11 @@ open class KotlinFileExtractor(
                     }
 
                     val javaResult = TypeResult(tw.getFreshIdLabel<DbClassorinterface>(), "", "")
-                    val kotlinResult = TypeResult(tw.getFreshIdLabel<DbKt_notnull_type>(), "", "")
-                    tw.writeKt_notnull_types(kotlinResult.id, javaResult.id)
+                    TODO() // TODO:KotType val kotlinResult = TypeResult(tw.getFreshIdLabel<DbKt_notnull_type>(), "", "")
+                    TODO() // TODO:KotType tw.writeKt_notnull_types(kotlinResult.id, javaResult.id)
                     val ids =
                         LocallyVisibleFunctionLabels(
-                            TypeResults(javaResult, kotlinResult),
+                            TODO(), // TODO:KotType TypeResults(javaResult, kotlinResult),
                             constructor = tw.getFreshIdLabel(),
                             constructorBlock = tw.getFreshIdLabel(),
                             function = tw.getFreshIdLabel()

java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt

@@ -126,13 +126,13 @@ open class KotlinUsesExtractor(
 
     private fun extractErrorType(): TypeResults {
         val javaResult = extractJavaErrorType()
-        val kotlinTypeId =
-            tw.getLabelFor<DbKt_nullable_type>("@\"errorKotlinType\"") {
-                tw.writeKt_nullable_types(it, javaResult.id)
-            }
+        TODO() // TODO:KotType val kotlinTypeId =
+        TODO() // TODO:KotType     tw.getLabelFor<DbKt_nullable_type>("@\"errorKotlinType\"") {
+        TODO() // TODO:KotType         tw.writeKt_nullable_types(it, javaResult.id)
+        TODO() // TODO:KotType     }
         return TypeResults(
             javaResult,
-            TypeResult(kotlinTypeId, "<CodeQL error type>", "<CodeQL error type>")
+            TODO() // TODO:KotType TypeResult(kotlinTypeId, "<CodeQL error type>", "<CodeQL error type>")
         )
     }
 
@@ -635,12 +635,12 @@ open class KotlinUsesExtractor(
                 "@\"FakeKotlinClass\"",
                 { tw.writeClasses_or_interfaces(it, "FakeKotlinClass", fakeKotlinPackageId, it) }
             )
-        val fakeKotlinTypeId: Label<DbKt_nullable_type> =
-            tw.getLabelFor(
-                "@\"FakeKotlinType\"",
-                { tw.writeKt_nullable_types(it, fakeKotlinClassId) }
-            )
-        return fakeKotlinTypeId
+        TODO() // TODO:KotType val fakeKotlinTypeId: Label<DbKt_nullable_type> =
+        TODO() // TODO:KotType     tw.getLabelFor(
+        TODO() // TODO:KotType         "@\"FakeKotlinType\"",
+        TODO() // TODO:KotType         { tw.writeKt_nullable_types(it, fakeKotlinClassId) }
+        TODO() // TODO:KotType     )
+        TODO() // TODO:KotType return fakeKotlinTypeId
     }
 
     // `args` can be null to describe a raw generic type.
@@ -659,15 +659,15 @@ open class KotlinUsesExtractor(
             else if (hasQuestionMark) {
                 val kotlinSignature = "$kotlinQualClassName?" // TODO: Is this right?
                 val kotlinLabel = "@\"kt_type;nullable;$kotlinQualClassName\""
-                val kotlinId: Label<DbKt_nullable_type> =
-                    tw.getLabelFor(kotlinLabel, { tw.writeKt_nullable_types(it, javaClassId) })
-                TypeResult(kotlinId, kotlinSignature, "TODO")
+                TODO() // TODO:KotType val kotlinId: Label<DbKt_nullable_type> =
+                TODO() // TODO:KotType     tw.getLabelFor(kotlinLabel, { tw.writeKt_nullable_types(it, javaClassId) })
+                TODO() // TODO:KotType TypeResult(kotlinId, kotlinSignature, "TODO")
             } else {
                 val kotlinSignature = kotlinQualClassName // TODO: Is this right?
                 val kotlinLabel = "@\"kt_type;notnull;$kotlinQualClassName\""
-                val kotlinId: Label<DbKt_notnull_type> =
-                    tw.getLabelFor(kotlinLabel, { tw.writeKt_notnull_types(it, javaClassId) })
-                TypeResult(kotlinId, kotlinSignature, "TODO")
+                TODO() // TODO:KotType val kotlinId: Label<DbKt_notnull_type> =
+                TODO() // TODO:KotType     tw.getLabelFor(kotlinLabel, { tw.writeKt_notnull_types(it, javaClassId) })
+                TODO() // TODO:KotType TypeResult(kotlinId, kotlinSignature, "TODO")
             }
         return TypeResults(javaResult, kotlinResult)
     }
@@ -864,19 +864,19 @@ open class KotlinUsesExtractor(
                     val kotlinSignature =
                         "$kotlinPackageName.$kotlinClassName?" // TODO: Is this right?
                     val kotlinLabel = "@\"kt_type;nullable;$kotlinPackageName.$kotlinClassName\""
-                    val kotlinId: Label<DbKt_nullable_type> =
-                        tw.getLabelFor(
-                            kotlinLabel,
-                            { tw.writeKt_nullable_types(it, kotlinClassId) }
-                        )
-                    TypeResult(kotlinId, kotlinSignature, "TODO")
+                    TODO() // TODO:KotType val kotlinId: Label<DbKt_nullable_type> =
+                    TODO() // TODO:KotType     tw.getLabelFor(
+                    TODO() // TODO:KotType         kotlinLabel,
+                    TODO() // TODO:KotType         { tw.writeKt_nullable_types(it, kotlinClassId) }
+                    TODO() // TODO:KotType     )
+                    TODO() // TODO:KotType TypeResult(kotlinId, kotlinSignature, "TODO")
                 } else {
                     val kotlinSignature =
                         "$kotlinPackageName.$kotlinClassName" // TODO: Is this right?
                     val kotlinLabel = "@\"kt_type;notnull;$kotlinPackageName.$kotlinClassName\""
-                    val kotlinId: Label<DbKt_notnull_type> =
-                        tw.getLabelFor(kotlinLabel, { tw.writeKt_notnull_types(it, kotlinClassId) })
-                    TypeResult(kotlinId, kotlinSignature, "TODO")
+                    TODO() // TODO:KotType val kotlinId: Label<DbKt_notnull_type> =
+                    TODO() // TODO:KotType     tw.getLabelFor(kotlinLabel, { tw.writeKt_notnull_types(it, kotlinClassId) })
+                    TODO() // TODO:KotType TypeResult(kotlinId, kotlinSignature, "TODO")
                 }
             return TypeResults(javaResult, kotlinResult)
         }
@@ -919,15 +919,15 @@ open class KotlinUsesExtractor(
                     else if (s.isNullable()) {
                         val kotlinSignature = "${javaResult.signature}?" // TODO: Wrong
                         val kotlinLabel = "@\"kt_type;nullable;type_param\"" // TODO: Wrong
-                        val kotlinId: Label<DbKt_nullable_type> =
-                            tw.getLabelFor(kotlinLabel, { tw.writeKt_nullable_types(it, aClassId) })
-                        TypeResult(kotlinId, kotlinSignature, "TODO")
+                        TODO() // TODO:KotType val kotlinId: Label<DbKt_nullable_type> =
+                        TODO() // TODO:KotType     tw.getLabelFor(kotlinLabel, { tw.writeKt_nullable_types(it, aClassId) })
+                        TODO() // TODO:KotType TypeResult(kotlinId, kotlinSignature, "TODO")
                     } else {
                         val kotlinSignature = javaResult.signature // TODO: Wrong
                         val kotlinLabel = "@\"kt_type;notnull;type_param\"" // TODO: Wrong
-                        val kotlinId: Label<DbKt_notnull_type> =
-                            tw.getLabelFor(kotlinLabel, { tw.writeKt_notnull_types(it, aClassId) })
-                        TypeResult(kotlinId, kotlinSignature, "TODO")
+                        TODO() // TODO:KotType val kotlinId: Label<DbKt_notnull_type> =
+                        TODO() // TODO:KotType     tw.getLabelFor(kotlinLabel, { tw.writeKt_notnull_types(it, aClassId) })
+                        TODO() // TODO:KotType TypeResult(kotlinId, kotlinSignature, "TODO")
                     }
                 return TypeResults(javaResult, kotlinResult)
             }
@@ -1621,8 +1621,8 @@ open class KotlinUsesExtractor(
         var res = tw.lm.locallyVisibleFunctionLabelMapping[f]
         if (res == null) {
             val javaResult = TypeResult(tw.getFreshIdLabel<DbClassorinterface>(), "", "")
-            val kotlinResult = TypeResult(tw.getFreshIdLabel<DbKt_notnull_type>(), "", "")
-            tw.writeKt_notnull_types(kotlinResult.id, javaResult.id)
+            val kotlinResult = TODO() // TODO:KotType TypeResult(tw.getFreshIdLabel<DbKt_notnull_type>(), "", "")
+            TODO() // TODO:KotType tw.writeKt_notnull_types(kotlinResult.id, javaResult.id)
             res =
                 LocallyVisibleFunctionLabels(
                     TypeResults(javaResult, kotlinResult),

java/kotlin-extractor2/.gitignore

@@ -0,0 +1,3 @@
+src/main/kotlin/KotlinExtractorDbScheme.kt
+!.idea
+out

java/kotlin-extractor2/.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

java/kotlin-extractor2/.idea/kotlinc.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Kotlin2JsCompilerArguments">
+    <option name="moduleKind" value="plain" />
+  </component>
+  <component name="Kotlin2JvmCompilerArguments">
+    <option name="jvmTarget" value="1.8" />
+  </component>
+  <component name="KotlinCommonCompilerArguments">
+    <option name="apiVersion" value="2.0" />
+    <option name="languageVersion" value="2.0" />
+  </component>
+  <component name="KotlinCompilerSettings">
+    <option name="additionalArguments" value="-Xcontext-receivers" />
+  </component>
+  <component name="KotlinJpsPluginSettings">
+    <option name="version" value="2.0.20" />
+  </component>
+</project>

java/kotlin-extractor2/.idea/libraries/jetbrains_kotlin_annotation_processing.xml

@@ -0,0 +1,14 @@
+<component name="libraryTable">
+  <library name="jetbrains.kotlin.annotation.processing" type="repository">
+    <properties maven-id="org.jetbrains.kotlin:kotlin-annotation-processing:2.1.0" />
+    <CLASSES>
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-annotation-processing/2.1.0/kotlin-annotation-processing-2.1.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-annotation-processing-compiler/2.1.0/kotlin-annotation-processing-compiler-2.1.0.jar!/" />
+    </CLASSES>
+    <JAVADOC />
+    <SOURCES>
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-annotation-processing/2.1.0/kotlin-annotation-processing-2.1.0-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-annotation-processing-compiler/2.1.0/kotlin-annotation-processing-compiler-2.1.0-sources.jar!/" />
+    </SOURCES>
+  </library>
+</component>

java/kotlin-extractor2/.idea/libraries/jetbrains_kotlin_compiler.xml

@@ -0,0 +1,30 @@
+<component name="libraryTable">
+  <library name="jetbrains.kotlin.compiler" type="repository">
+    <properties maven-id="org.jetbrains.kotlin:kotlin-compiler:2.1.0" />
+    <CLASSES>
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-compiler/2.1.0/kotlin-compiler-2.1.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-stdlib-jdk8/2.1.0/kotlin-stdlib-jdk8-2.1.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-stdlib/2.1.0/kotlin-stdlib-2.1.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/annotations/13.0/annotations-13.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-stdlib-jdk7/2.1.0/kotlin-stdlib-jdk7-2.1.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-script-runtime/2.1.0/kotlin-script-runtime-2.1.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-reflect/1.6.10/kotlin-reflect-1.6.10.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/intellij/deps/trove4j/1.0.20200330/trove4j-1.0.20200330.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlinx/kotlinx-coroutines-core-jvm/1.6.4/kotlinx-coroutines-core-jvm-1.6.4.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-stdlib-common/1.6.21/kotlin-stdlib-common-1.6.21.jar!/" />
+    </CLASSES>
+    <JAVADOC />
+    <SOURCES>
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-compiler/2.1.0/kotlin-compiler-2.1.0-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-stdlib-jdk8/2.1.0/kotlin-stdlib-jdk8-2.1.0-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-stdlib/2.1.0/kotlin-stdlib-2.1.0-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/annotations/13.0/annotations-13.0-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-stdlib-jdk7/2.1.0/kotlin-stdlib-jdk7-2.1.0-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-script-runtime/2.1.0/kotlin-script-runtime-2.1.0-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-reflect/1.6.10/kotlin-reflect-1.6.10-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/intellij/deps/trove4j/1.0.20200330/trove4j-1.0.20200330-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlinx/kotlinx-coroutines-core-jvm/1.6.4/kotlinx-coroutines-core-jvm-1.6.4-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/kotlin/kotlin-stdlib-common/1.6.21/kotlin-stdlib-common-1.6.21-sources.jar!/" />
+    </SOURCES>
+  </library>
+</component>

java/kotlin-extractor2/.idea/misc.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_17" default="true" project-jdk-name="temurin-17" project-jdk-type="JavaSDK">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>

java/kotlin-extractor2/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/kotlin-extractor2.iml" filepath="$PROJECT_DIR$/kotlin-extractor2.iml" />
+    </modules>
+  </component>
+</project>

java/kotlin-extractor2/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$/../.." vcs="Git" />
+  </component>
+</project>

java/kotlin-extractor2/BUILD.bazel

@@ -0,0 +1,65 @@
+load("@rules_kotlin//kotlin:core.bzl", "kt_javac_options", "kt_kotlinc_options")
+load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library")
+load("//misc/bazel:pkg.bzl", "codeql_pkg_files")
+
+kt_javac_options(
+    name = "javac-options",
+    # if needed, see https://bazelbuild.github.io/rules_kotlin/kotlin.html#kt_javac_options for available options
+)
+
+kt_kotlinc_options(
+    name = "kotlinc-options",
+    x_context_receivers = True,
+    # TODO:
+    # warn = "error",
+    # if needed, see https://bazelbuild.github.io/rules_kotlin/kotlin.html#kt_kotlinc_options for available options
+)
+
+py_binary(
+    name = "generate_dbscheme",
+    srcs = ["generate_dbscheme.py"],
+)
+
+genrule(
+    name = "generated-dbscheme",
+    srcs = ["@codeql//java:dbscheme"],
+    outs = ["KotlinExtractorDbScheme.kt"],
+    cmd = "$(execpath :generate_dbscheme) $< $@",
+    tools = [":generate_dbscheme"],
+)
+
+kt_jvm_library(
+    name = "ke2-kt",
+    srcs = [s for s in [":generated-dbscheme"] + glob(["src/main/java/**/*.java"]) + glob(["src/main/kotlin/**/*.kt"]) if s != "src/main/kotlin/KotlinExtractorDbScheme.kt"],
+    javac_opts = ":javac-options",
+    kotlinc_opts = ":kotlinc-options",
+    module_name = "codeql-kotlin-extractor2",
+    deps = [
+        "@maven_deps//:org_jetbrains_kotlin_%s" % kotlin_lib
+        for kotlin_lib in (
+            "kotlin_annotation_processing",
+            "kotlin_compiler",
+        )
+    ] + ["@maven_deps//:com_github_ben_manes_caffeine_caffeine"],
+)
+
+java_binary(
+    name = "ke2-java",
+    runtime_deps = [":ke2-kt"],
+)
+
+codeql_pkg_files(
+    name = "kotlin-extractor2",
+    srcs = [
+        ":ke2-java_deploy.jar",
+    ],
+    exes = [
+        "ke2.sh",
+        "ke2.cmd",
+    ],
+    renames = {
+        "ke2.sh": "ke2",
+        ":ke2-java_deploy.jar": "ke2.jar",
+    },
+    visibility = ["//visibility:public"],
+)

java/kotlin-extractor2/build.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+import subprocess
+
+subprocess.run(["kotlinc", "src/main/kotlin/KotlinExtractor.kt"])
+

java/kotlin-extractor2/defaults/BUILD.bazel

@@ -0,0 +1,30 @@
+load("@codeql_kotlin_defaults//:defaults.bzl", "kotlin_extractor_defaults")
+
+package(default_visibility = ["//java/kotlin-extractor:__pkg__"])
+
+_common_extractor_name_prefix = "codeql-extractor-kotlin"
+
+alias(
+    name = "%s-standalone" % _common_extractor_name_prefix,
+    actual = "//java/kotlin-extractor:%s-standalone-%s" % (
+        _common_extractor_name_prefix,
+        kotlin_extractor_defaults.extractor_version,
+    ),
+)
+
+alias(
+    name = "%s-embeddable" % _common_extractor_name_prefix,
+    actual = "//java/kotlin-extractor:%s-embeddable-%s" % (
+        _common_extractor_name_prefix,
+        kotlin_extractor_defaults.extractor_version,
+    ),
+)
+
+alias(
+    name = _common_extractor_name_prefix,
+    actual = "//java/kotlin-extractor:%s-%s-%s" % (
+        _common_extractor_name_prefix,
+        kotlin_extractor_defaults.variant,
+        kotlin_extractor_defaults.extractor_version,
+    ),
+)

java/kotlin-extractor2/dev/.gitignore

@@ -0,0 +1 @@
+/.kotlinc_*

java/kotlin-extractor2/dev/kotlin

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec -a "$0" "$(dirname "$0")/wrapper.py" kotlin "$@"

java/kotlin-extractor2/dev/kotlin.bat

@@ -0,0 +1,4 @@
+@echo off
+
+python "%~dp0wrapper.py" kotlin %*
+exit /b %ERRORLEVEL%

java/kotlin-extractor2/dev/kotlinc

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec -a "$0" "$(dirname "$0")/wrapper.py" kotlinc "$@"

java/kotlin-extractor2/dev/kotlinc.bat

@@ -0,0 +1,4 @@
+@echo off
+
+python "%~dp0wrapper.py" kotlinc %*
+exit /b %ERRORLEVEL%

java/kotlin-extractor2/dev/wrapper.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+
+"""
+Wrapper script that manages kotlin versions.
+Usage: add this directory to your PATH, then
+* `kotlin* --select x.y.z` will select the version for the next invocations, checking it actually exists
+* `kotlin* --clear` will remove any state of the wrapper (deselecting a previous version selection)
+* `kotlinc -version` will print the selected version information. It will not print `JRE` information as a normal
+  `kotlinc` invocation would do though. In exchange, the invocation incurs no overhead.
+* Any other invocation will forward to the selected kotlin tool version, downloading it if necessary. If no version was
+  previously selected with `--select`, a default will be used (see `DEFAULT_VERSION` below)
+
+In order to install kotlin, ripunzip will be used if installed, or if running on Windows within `semmle-code` (ripunzip
+is available in `resources/lib/windows/ripunzip` then).
+"""
+
+import pathlib
+import urllib
+import urllib.request
+import urllib.error
+import argparse
+import sys
+import platform
+import subprocess
+import zipfile
+import shutil
+import io
+import os
+
+DEFAULT_VERSION = "2.0.0"
+
+
+def options():
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("tool")
+    parser.add_argument("--select")
+    parser.add_argument("--clear", action="store_true")
+    parser.add_argument("-version", action="store_true")
+    return parser.parse_known_args()
+
+
+file_template = "kotlin-compiler-{version}.zip"
+url_template = "https://github.com/JetBrains/kotlin/releases/download/v{version}/kotlin-compiler-{version}.zip"
+this_dir = pathlib.Path(__file__).resolve().parent
+version_file = this_dir / ".kotlinc_version"
+install_dir = this_dir / ".kotlinc_installed"
+zips_dir = this_dir / ".kotlinc_zips"
+windows_ripunzip = (
+    this_dir.parents[4] / "resources" / "lib" / "windows" / "ripunzip" / "ripunzip.exe"
+)
+
+
+class Error(Exception):
+    pass
+
+
+class ZipFilePreservingPermissions(zipfile.ZipFile):
+    def _extract_member(self, member, targetpath, pwd):
+        if not isinstance(member, zipfile.ZipInfo):
+            member = self.getinfo(member)
+
+        targetpath = super()._extract_member(member, targetpath, pwd)
+
+        attr = member.external_attr >> 16
+        if attr != 0:
+            os.chmod(targetpath, attr)
+        return targetpath
+
+
+def get_version():
+    try:
+        return version_file.read_text()
+    except FileNotFoundError:
+        return None
+
+
+def install(version: str, quiet: bool):
+    if quiet:
+        info_out = subprocess.DEVNULL
+        info = lambda *args: None
+    else:
+        info_out = sys.stderr
+        info = lambda *args: print(*args, file=sys.stderr)
+    file = file_template.format(version=version)
+    url = url_template.format(version=version)
+    if install_dir.exists():
+        shutil.rmtree(install_dir)
+    install_dir.mkdir()
+    zips_dir.mkdir(exist_ok=True)
+    zip = zips_dir / file
+
+    if not zip.exists():
+        info(f"downloading {url}")
+        tmp_zip = zip.with_suffix(".tmp")
+        with open(tmp_zip, "wb") as out, urllib.request.urlopen(url) as response:
+            shutil.copyfileobj(response, out)
+        tmp_zip.rename(zip)
+    ripunzip = shutil.which("ripunzip")
+    if (
+        ripunzip is None
+        and platform.system() == "Windows"
+        and windows_ripunzip.exists()
+    ):
+        ripunzip = windows_ripunzip
+    if ripunzip:
+        info(f"extracting {zip} using ripunzip")
+        subprocess.run(
+            [ripunzip, "unzip-file", zip],
+            stdout=info_out,
+            stderr=info_out,
+            cwd=install_dir,
+            check=True,
+        )
+    else:
+        info(f"extracting {zip}")
+        with ZipFilePreservingPermissions(zip) as archive:
+            archive.extractall(install_dir)
+
+
+def forward(tool, forwarded_opts):
+    tool = install_dir / "kotlinc" / "bin" / tool
+    if platform.system() == "Windows":
+        tool = tool.with_suffix(".bat")
+    assert tool.exists(), f"{tool} not found"
+    cmd = [tool] + forwarded_opts
+    if platform.system() == "Windows":
+        # kotlin bat script is pretty sensible to unquoted args on windows
+        ret = subprocess.run(" ".join(f'"{a}"' for a in cmd)).returncode
+        sys.exit(ret)
+    else:
+        os.execv(cmd[0], cmd)
+
+
+def clear():
+    if install_dir.exists():
+        print(f"removing {install_dir}", file=sys.stderr)
+        shutil.rmtree(install_dir)
+    if version_file.exists():
+        print(f"removing {version_file}", file=sys.stderr)
+        version_file.unlink()
+    if zips_dir.exists():
+        print(f"removing {zips_dir}", file=sys.stderr)
+        shutil.rmtree(zips_dir)
+
+
+def main(opts, forwarded_opts):
+    if opts.clear:
+        clear()
+        return
+    current_version = get_version()
+    if opts.select == "default":
+        selected_version = DEFAULT_VERSION
+    elif opts.select is not None:
+        selected_version = opts.select
+    else:
+        selected_version = current_version or DEFAULT_VERSION
+    if selected_version != current_version:
+        # don't print information about install procedure unless explicitly using --select
+        install(selected_version, quiet=opts.select is None)
+        version_file.write_text(selected_version)
+    if opts.select and not forwarded_opts and not opts.version:
+        print(f"selected {selected_version}")
+        return
+    if opts.version:
+        if opts.tool == "kotlinc":
+            print(
+                f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)",
+                file=sys.stderr,
+            )
+            return
+        forwarded_opts.append("-version")
+
+    forward(opts.tool, forwarded_opts)
+
+
+if __name__ == "__main__":
+    try:
+        main(*options())
+    except Error as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+    except KeyboardInterrupt:
+        sys.exit(1)

java/kotlin-extractor2/generate_dbscheme.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+
+import re
+import sys
+
+enums = {}
+unions = {}
+tables = {}
+
+dbscheme = sys.argv[1] if len(sys.argv) >= 2 else '../ql/lib/config/semmlecode.dbscheme'
+output = sys.argv[2] if len(sys.argv) >= 3 else 'src/main/kotlin/KotlinExtractorDbScheme.kt'
+
+def parse_dbscheme(filename):
+    with open(filename, 'r') as f:
+        dbscheme = f.read()
+
+    # Remove comments
+    dbscheme = re.sub(r'/\*.*?\*/', '', dbscheme, flags=re.DOTALL)
+    dbscheme = re.sub(r'//[^\r\n]*', '', dbscheme)
+
+    # kind enums
+    for name, kind, body in re.findall(r'case\s+@([^.\s]*)\.([^.\s]*)\s+of\b(.*?);',
+                                       dbscheme,
+                                       flags=re.DOTALL):
+        mapping = []
+        for num, typ in re.findall(r'(\d+)\s*=\s*@(\S+)', body):
+            mapping.append((int(num), typ))
+        enums[name] = (kind, mapping)
+
+    # unions
+    for name, rhs in re.findall(r'@(\w+)\s*=\s*(@\w+(?:\s*\|\s*@\w+)*)',
+                                dbscheme,
+                                flags=re.DOTALL):
+        typs = re.findall(r'@(\w+)', rhs)
+        unions[name] = typs
+
+    # tables
+    for relname, body in re.findall('\n([\w_]+)(\([^)]*\))',
+                                    dbscheme,
+                                    flags=re.DOTALL):
+        columns = list(re.findall('(\S+)\s*:\s*([^\s,]+)(?:\s+(ref)|)', body))
+        tables[relname] = columns
+
+parse_dbscheme(dbscheme)
+
+type_aliases = {}
+
+for alias, typs in unions.items():
+    if len(typs) == 1:
+        real = typs[0]
+        if real in type_aliases:
+            real = type_aliases[real]
+        type_aliases[alias] = real
+
+def unalias(t):
+    return type_aliases.get(t, t)
+
+type_leaf = set()
+type_union = {}
+
+for name, (kind, mapping) in enums.items():
+    s = set()
+    for num, typ in mapping:
+        s.add(typ)
+        type_leaf.add(typ)
+    type_union[name] = s
+
+for name, typs in unions.items():
+    if name not in type_aliases:
+        type_union[name] = set(map(unalias, typs))
+
+for relname, columns in tables.items():
+    for _, db_type, ref in columns:
+        if db_type[0] == '@' and ref == '':
+            db_type_name = db_type[1:]
+            if db_type_name not in enums:
+                type_leaf.add(db_type_name)
+
+type_union_of_leaves = {}
+
+def to_leaves(t):
+    if t not in type_union_of_leaves:
+        xs = type_union[t]
+        leaves = set()
+        for x in xs:
+            if x in type_leaf:
+                leaves.add(x)
+            else:
+                to_leaves(x)
+                leaves.update(type_union_of_leaves[x])
+        type_union_of_leaves[t] = leaves
+
+for t in type_union:
+    to_leaves(t)
+
+supertypes = {}
+for t in type_leaf:
+    supers = set()
+    for sup, s in type_union_of_leaves.items():
+        if t in s:
+            supers.add(sup)
+    supertypes[t] = supers
+for t, leaves in type_union_of_leaves.items():
+    supers = set()
+    for sup, s in type_union_of_leaves.items():
+        if t != sup and leaves.issubset(s):
+            supers.add(sup)
+    supertypes[t] = supers
+
+def upperFirst(string):
+    return string[0].upper() + string[1:]
+
+def genTable(kt, relname, columns, enum = None, kind = None, num = None, typ = None):
+    kt.write('fun TrapWriter.write' + upperFirst(relname))
+    if kind is not None:
+        kt.write('_' + typ)
+    kt.write('(')
+    for colname, db_type, _ in columns:
+        if colname != kind:
+            kt.write(colname + ': ')
+            if db_type == 'int':
+                kt.write('Int')
+            elif db_type == 'float':
+                kt.write('Double')
+            elif db_type == 'string':
+                kt.write('String')
+            elif db_type == 'date':
+                kt.write('Date')
+            elif db_type == 'boolean':
+                kt.write('Boolean')
+            elif db_type[0] == '@':
+                label = db_type[1:]
+                if label == enum:
+                    label = typ
+                kt.write('Label<out Db' + upperFirst(label) + '>')
+            else:
+                raise Exception('Bad db_type: ' + db_type)
+            kt.write(', ')
+    kt.write(') {\n')
+    kt.write('    this.writeTrap("' + relname + '(')
+    comma = ''
+    for colname, db_type, _ in columns:
+        kt.write(comma)
+        if colname == kind:
+            kt.write(str(num))
+        elif db_type == 'string':
+            kt.write('\\"${this.escapeTrapString(this.truncateString(' + colname + '))}\\"')
+        elif db_type == 'date':
+            kt.write('D\\"${' + colname + '}\\"')
+        else:
+            kt.write('$' + colname)
+        comma = ', '
+    kt.write(')\\n")\n')
+    kt.write('}\n')
+
+with open(output, 'w') as kt:
+    kt.write('/* Generated by ' + sys.argv[0] + ': Do not edit manually. */\n')
+    kt.write('package com.github.codeql\n')
+    kt.write('import java.util.Date\n')
+
+    for relname, columns in tables.items():
+        enum = None
+        for _, db_type, ref in columns:
+            if db_type[0] == '@' and ref == '':
+                db_type_name = db_type[1:]
+                if db_type_name in enums:
+                    enum = db_type_name
+        if enum is None:
+            genTable(kt, relname, columns)
+        else:
+            (kind, mapping) = enums[enum]
+            for num, typ in mapping:
+                genTable(kt, relname, columns, enum, kind, num, typ)
+
+    kt.write('sealed interface AnyDbType\n')
+    for typ in sorted(supertypes):
+        kt.write('sealed interface Db' + upperFirst(typ) + ': AnyDbType')
+        # Sorting makes the output deterministic.
+        names = sorted(supertypes[typ])
+        kt.write(''.join(map(lambda name: ', Db' + upperFirst(name), names)))
+        kt.write('\n')
+    for alias in sorted(type_aliases):
+        kt.write('typealias Db' + upperFirst(alias) + ' = Db' + upperFirst(type_aliases[alias]) + '\n')

java/kotlin-extractor2/ke2.cmd

@@ -0,0 +1,10 @@
+@echo off
+
+IF [%CODEQL_JAVA_HOME] == [] (
+  set JAVA=java.exe
+) else (
+  set JAVA=%CODEQL_JAVA_HOME\bin\java.exe
+)
+
+%JAVA -cp %~dp0ke2_deploy.jar KotlinExtractorKt %*
+exit /b %ERRORLEVEL%

java/kotlin-extractor2/ke2.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(dirname "$0")"
+
+# This default should be kept in sync with
+# com.semmle.extractor.java.interceptors.KotlinInterceptor.initializeExtractionContext
+TRAP_DIR="$CODEQL_EXTRACTOR_JAVA_TRAP_DIR"
+if [ "$TRAP_DIR" = "" ]
+then
+    TRAP_DIR="kotlin-extractor/trap"
+fi
+mkdir -p "$TRAP_DIR"
+
+INVOCATION_TRAP=`mktemp -p "$TRAP_DIR" invocation.XXXXXXXXXX.trap`
+
+echo "// Invocation of Kotlin Extractor 2" >> "$INVOCATION_TRAP"
+echo "#compilation = *" >> "$INVOCATION_TRAP"
+# TODO: This should be properly escaped:
+echo "compilations(#compilation, 2, \"`pwd`\",\"$INVOCATION_TRAP\")" >> "$INVOCATION_TRAP"
+ARG_INDEX=0
+for ARG in "$@"
+do
+    # TODO: This should be properly escaped:
+    echo "compilation_args(#compilation, $ARG_INDEX, \"$ARG\")" >> "$INVOCATION_TRAP"
+    ARG_INDEX=$(("$ARG_INDEX" + 1))
+done
+
+if [[ -n "$CODEQL_JAVA_HOME" ]]; then
+  JAVA="$CODEQL_JAVA_HOME/bin/java"
+else
+  JAVA=java
+fi
+
+"$JAVA" -Xmx2G $CODEQL_KOTLIN_EXTRACTOR_JVM_ARGS -cp "$SCRIPT_DIR/ke2.jar" com.github.codeql.KotlinExtractorKt "$INVOCATION_TRAP" "$@"

java/kotlin-extractor2/kotlin-extractor2.iml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/main/kotlin" isTestSource="false" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="jetbrains.kotlin.compiler" level="project" />
+    <orderEntry type="library" name="jetbrains.kotlin.annotation.processing" level="project" />
+  </component>
+</module>

java/kotlin-extractor2/pick-kotlin-version.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""
+Script to get currently installed kotlinc version. If a list of available versions is provided as input,
+the last version of those lower or equal to the kotlinc version is printed.
+"""
+
+import subprocess
+import re
+import shutil
+import argparse
+import sys
+
+
+def version_tuple(v):
+    v, _, _ = v.partition('-')
+    return tuple(int(x) for x in v.split(".", 2))
+
+
+p = argparse.ArgumentParser(description=__doc__, fromfile_prefix_chars='@')
+p.add_argument("available_versions", nargs="*", metavar="X.Y.Z")
+opts = p.parse_args()
+
+kotlinc = shutil.which('kotlinc')
+if kotlinc is None:
+    raise Exception("kotlinc not found")
+res = subprocess.run([kotlinc, "-version"], text=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+if res.returncode != 0:
+    raise Exception(f"kotlinc -version failed: {res.stderr}")
+m = re.match(r'.* kotlinc-jvm ([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z][a-zA-Z0-9]*)?) .*', res.stderr)
+if m is None:
+    raise Exception(f'Cannot detect version of kotlinc (got {res.stderr})')
+version = m[1]
+if opts.available_versions:
+    vt = version_tuple(version)
+    available = sorted(opts.available_versions, key=version_tuple, reverse=True)
+    for v in available:
+        if version_tuple(v) <= vt:
+            print(v)
+            sys.exit(0)
+    raise Exception(f'Cannot find an available version for {version}')
+print(version)

java/kotlin-extractor2/src/main/java/com/semmle/extractor/java/OdasaOutput.java

@@ -0,0 +1,746 @@
+package com.semmle.extractor.java;
+
+import java.lang.reflect.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+import com.github.codeql.Logger;
+// TODO
+//import static com.github.codeql.ClassNamesKt.getIrElementBinaryName;
+//import static com.github.codeql.ClassNamesKt.getIrClassVirtualFile;
+
+import org.jetbrains.kotlin.analysis.api.symbols.KaClassSymbol;
+import org.jetbrains.kotlin.analysis.api.symbols.KaSymbol;
+
+import com.intellij.openapi.vfs.VirtualFile;
+
+import org.jetbrains.org.objectweb.asm.ClassVisitor;
+import org.jetbrains.org.objectweb.asm.ClassReader;
+import org.jetbrains.org.objectweb.asm.Opcodes;
+
+import com.semmle.util.concurrent.LockDirectory;
+import com.semmle.util.concurrent.LockDirectory.LockingMode;
+import com.semmle.util.data.Pair;
+import com.semmle.util.exception.CatastrophicError;
+import com.semmle.util.exception.NestedError;
+import com.semmle.util.exception.ResourceError;
+import com.semmle.util.extraction.SpecFileEntry;
+import com.semmle.util.files.FileUtil;
+import com.semmle.util.io.WholeIO;
+import com.semmle.util.process.Env;
+import com.semmle.util.process.Env.Var;
+import com.semmle.util.trap.dependencies.TrapDependencies;
+import com.semmle.util.trap.dependencies.TrapSet;
+import com.semmle.util.trap.pathtransformers.PathTransformer;
+
+import com.github.codeql.Compression;
+
+import static com.github.codeql.ClassNamesKt.getSymbolBinaryName;
+
+public class OdasaOutput {
+	private final File trapFolder;
+	private final File sourceArchiveFolder;
+
+	private File currentSourceFile;
+	private TrapSet trapsCreated;
+	private TrapDependencies trapDependenciesForSource;
+
+	private SpecFileEntry currentSpecFileEntry;
+
+	// should origin tracking be used?
+	private final boolean trackClassOrigins;
+
+	private final Logger log;
+	private final Compression compression;
+
+	/**
+	 * DEBUG only: just use the given file as the root for TRAP, source archive etc
+	 */
+	OdasaOutput(File outputRoot, Compression compression, Logger log) {
+		this.trapFolder = new File(outputRoot, "trap");
+		this.sourceArchiveFolder = new File(outputRoot, "src_archive");
+		this.trackClassOrigins = false;
+		this.log = log;
+		this.compression = compression;
+	}
+
+	public OdasaOutput(boolean trackClassOrigins, Compression compression, Logger log) {
+		String trapFolderVar = Env.systemEnv().get("CODEQL_EXTRACTOR_JAVA_TRAP_DIR");
+		if (trapFolderVar == null) {
+			throw new ResourceError("CODEQL_EXTRACTOR_JAVA_TRAP_DIR was not set");
+		}
+		String sourceArchiveVar = Env.systemEnv().get("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR");
+		if (sourceArchiveVar == null) {
+			throw new ResourceError("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR was not set");
+		}
+		this.trapFolder = new File(trapFolderVar);
+		this.sourceArchiveFolder = new File(sourceArchiveVar);
+		this.trackClassOrigins = trackClassOrigins;
+		this.log = log;
+		this.compression = compression;
+	}
+
+	public File getTrapFolder() {
+		return trapFolder;
+	}
+
+	public boolean getTrackClassOrigins() {
+		return trackClassOrigins;
+	}
+
+	/**
+	 * Set the source file that is currently being processed. This may affect
+	 * things like trap and source archive directories, and persists as a
+	 * setting until this method is called again.
+	 *
+	 * @param f the current source file
+	 */
+	public void setCurrentSourceFile(File f) {
+		currentSourceFile = f;
+		currentSpecFileEntry = entryFor();
+		trapsCreated = new TrapSet();
+		trapsCreated.addSource(PathTransformer.std().fileAsDatabaseString(f));
+		trapDependenciesForSource = null;
+	}
+
+	/** The output paths for that file, or null if it shouldn't be included */
+	private SpecFileEntry entryFor() {
+		return new SpecFileEntry(trapFolder, sourceArchiveFolder,
+				Arrays.asList(PathTransformer.std().fileAsDatabaseString(currentSourceFile)));
+	}
+
+	/*
+	 * Trap sets and dependencies.
+	 */
+
+	public void writeTrapSet() {
+		trapsCreated.save(trapSetFor(currentSourceFile).toPath());
+	}
+
+	private File trapSetFor(File file) {
+		return FileUtil.appendAbsolutePath(
+				currentSpecFileEntry.getTrapFolder(), PathTransformer.std().fileAsDatabaseString(file) + ".set");
+	}
+
+	public void addDependency(KaSymbol sym, String signature) {
+		String path = trapFilePathForDecl(sym, signature);
+		trapDependenciesForSource.addDependency(path);
+	}
+
+	/*
+	 * Source archive.
+	 */
+
+	/**
+	 * Write the given source file to the right source archive, encoded in UTF-8,
+	 * or do nothing if the file shouldn't be populated.
+	 */
+	public void writeCurrentSourceFileToSourceArchive(String contents) {
+		if (currentSpecFileEntry != null && currentSpecFileEntry.getSourceArchivePath() != null) {
+			File target = sourceArchiveFileFor(currentSourceFile);
+			target.getParentFile().mkdirs();
+			new WholeIO().write(target, contents);
+		}
+	}
+
+	public void writeFileToSourceArchive(File srcFile) {
+		File target = sourceArchiveFileFor(srcFile);
+		target.getParentFile().mkdirs();
+		String contents = new WholeIO().strictread(srcFile);
+		new WholeIO().write(target, contents);
+	}
+
+	private File sourceArchiveFileFor(File file) {
+		return FileUtil.appendAbsolutePath(currentSpecFileEntry.getSourceArchivePath(),
+				PathTransformer.std().fileAsDatabaseString(file));
+	}
+
+	/*
+	 * Trap file names and paths.
+	 */
+
+	private static final String CLASSES_DIR = "classes";
+	private static final String JARS_DIR = "jars";
+	private static final String MODULES_DIR = "modules";
+
+	private File getTrapFileForCurrentSourceFile() {
+		if (currentSpecFileEntry == null)
+			return null;
+		return trapFileFor(currentSourceFile);
+	}
+
+	private File getTrapFileForJarFile(File jarFile) {
+		if (!jarFile.getAbsolutePath().endsWith(".jar"))
+			return null;
+		return FileUtil.appendAbsolutePath(
+				currentSpecFileEntry.getTrapFolder(),
+				JARS_DIR + "/" + PathTransformer.std().fileAsDatabaseString(jarFile) + ".trap"
+						+ compression.getExtension());
+	}
+
+	private File getTrapFileForModule(String moduleName) {
+		return FileUtil.appendAbsolutePath(
+				currentSpecFileEntry.getTrapFolder(),
+				MODULES_DIR + "/" + moduleName + ".trap" + compression.getExtension());
+	}
+
+	private File trapFileFor(File file) {
+		return FileUtil.appendAbsolutePath(currentSpecFileEntry.getTrapFolder(),
+				PathTransformer.std().fileAsDatabaseString(file) + ".trap" + compression.getExtension());
+	}
+
+	private File getTrapFileForDecl(KaSymbol sym, String signature) {
+		if (currentSpecFileEntry == null)
+			return null;
+		return trapFileForDecl(sym, signature);
+	}
+
+	private File trapFileForDecl(KaSymbol sym, String signature) {
+		return FileUtil.fileRelativeTo(currentSpecFileEntry.getTrapFolder(),
+				trapFilePathForDecl(sym, signature));
+	}
+
+	private String trapFilePathForDecl(KaSymbol sym, String signature) {
+		String binaryName = getSymbolBinaryName(sym);
+		// TODO: Reinstate this?
+		// if (getTrackClassOrigins())
+		// classId += "-" + StringDigestor.digest(sym.getSourceFileId());
+		String result = CLASSES_DIR + "/" +
+		binaryName.replace('.', '/') +
+		signature +
+		".members" +
+		".trap" + compression.getExtension();
+		return result;
+	}
+
+	/*
+	 * Trap writers.
+	 */
+
+	/**
+	 * Get a {@link TrapFileManager} to write members
+	 * about a declaration, or <code>null</code> if the declaration shouldn't be
+	 * populated.
+	 *
+	 * @param sym
+	 *                  The declaration's symbol, including, in particular, its
+	 *                  fully qualified
+	 *                  binary class name.
+	 * @param signature
+	 *                  Any unique suffix needed to distinguish `sym` from other
+	 *                  declarations with the same name.
+	 *                  For functions for example, this means its parameter
+	 *                  signature.
+	 */
+	private TrapFileManager getMembersWriterForDecl(File trap, File trapFileBase, TrapClassVersion trapFileVersion,
+                                                  KaSymbol sym, String signature) {
+		// If the TRAP file already exists then we
+		// don't need to write it.
+		if (trap.exists()) {
+			log.trace("Not rewriting trap file for " + trap.toString() + " as it exists");
+			return null;
+		}
+		// If the TRAP file was written in the past, and
+		// then renamed to its trap-old name, then we
+		// don't need to rewrite it only to rename it
+		// again.
+		File trapFileDir = trap.getParentFile();
+		File trapOld = new File(trapFileDir,
+				trap.getName().replace(".trap" + compression.getExtension(), ".trap-old" + compression.getExtension()));
+		if (trapOld.exists()) {
+			log.trace("Not rewriting trap file for " + trap.toString() + " as the trap-old exists");
+			return null;
+		}
+		// Otherwise, if any newer TRAP file has already
+		// been written then we don't need to write
+		// anything.
+		if (trapFileBase != null && trapFileVersion != null && trapFileDir.exists()) {
+			String trapFileBaseName = trapFileBase.getName();
+
+			for (File f : FileUtil.list(trapFileDir)) {
+				String name = f.getName();
+				Matcher m = selectClassVersionComponents.matcher(name);
+				if (m.matches() && m.group(1).equals(trapFileBaseName)) {
+					TrapClassVersion v = new TrapClassVersion(Integer.valueOf(m.group(2)), Integer.valueOf(m.group(3)),
+							Long.valueOf(m.group(4)), m.group(5));
+					if (v.newerThan(trapFileVersion)) {
+						log.trace("Not rewriting trap file for " + trap.toString() + " as " + f.toString() + " exists");
+						return null;
+					}
+				}
+			}
+		}
+		return trapWriter(trap, sym, signature);
+	}
+
+	private TrapFileManager trapWriter(File trapFile, KaSymbol sym, String signature) {
+		if (!trapFile.getName().endsWith(".trap" + compression.getExtension()))
+			throw new CatastrophicError("OdasaOutput only supports writing to compressed trap files");
+		String relative = FileUtil.relativePath(trapFile, currentSpecFileEntry.getTrapFolder());
+		trapFile.getParentFile().mkdirs();
+		trapsCreated.addTrap(relative);
+		return concurrentWriter(trapFile, relative, log, sym, signature);
+	}
+
+	private TrapFileManager concurrentWriter(File trapFile, String relative, Logger log, KaSymbol sym,
+			String signature) {
+		if (trapFile.exists())
+			return null;
+		return new TrapFileManager(trapFile, relative, true, log, sym, signature);
+	}
+
+	public class TrapFileManager implements AutoCloseable {
+
+		private TrapDependencies trapDependenciesForClass;
+		private File trapFile;
+		private KaSymbol sym;
+		private String signature;
+		private boolean hasError = false;
+
+		private TrapFileManager(File trapFile, String relative, boolean concurrentCreation, Logger log, KaSymbol sym,
+				String signature) {
+			trapDependenciesForClass = new TrapDependencies(relative);
+			this.trapFile = trapFile;
+			this.sym = sym;
+			this.signature = signature;
+		}
+
+		public File getFile() {
+			return trapFile;
+		}
+
+		public void addDependency(KaSymbol dep, String signature) {
+			trapDependenciesForClass.addDependency(trapFilePathForDecl(dep, signature));
+		}
+
+		public void addDependency(KaClassSymbol c) {
+			addDependency(c, "");
+		}
+
+		public void close() {
+			if (hasError) {
+				return;
+			}
+
+			writeTrapDependencies(trapDependenciesForClass);
+		}
+
+		private void writeTrapDependencies(TrapDependencies trapDependencies) {
+			String dep = trapDependencies.trapFile().replace(".trap" + compression.getExtension(), ".dep");
+			trapDependencies.save(
+					currentSpecFileEntry.getTrapFolder().toPath().resolve(dep));
+		}
+
+		public void setHasError() {
+			hasError = true;
+		}
+	}
+
+	/*
+	 * Trap file locking.
+	 */
+
+	private final Pattern selectClassVersionComponents = Pattern
+			.compile("(.*)#(-?[0-9]+)\\.(-?[0-9]+)-(-?[0-9]+)-(.*)\\.trap.*");
+
+	/**
+	 * <b>CAUTION</b>: to avoid the potential for deadlock between multiple
+	 * concurrent extractor processes,
+	 * only one source file {@link TrapLocker} may be open at any time, and the lock
+	 * must be obtained
+	 * <b>before</b> any <b>class</b> file lock.
+	 *
+	 * Trap file extensions (and paths) ensure that source and class file locks are
+	 * distinct.
+	 *
+	 * @return a {@link TrapLocker} for the currently processed source file, which
+	 *         must have been
+	 *         previously set by a call to
+	 *         {@link OdasaOutput#setCurrentSourceFile(File)}.
+	 */
+	public TrapLocker getTrapLockerForCurrentSourceFile() {
+		return new TrapLocker((KaClassSymbol) null, null, true);
+	}
+
+	/**
+	 * <b>CAUTION</b>: to avoid the potential for deadlock between multiple
+	 * concurrent extractor processes,
+	 * only one jar file {@link TrapLocker} may be open at any time, and the lock
+	 * must be obtained
+	 * <b>after</b> any <b>source</b> file lock. Only one jar or class file lock may
+	 * be open at any time.
+	 *
+	 * Trap file extensions (and paths) ensure that source and jar file locks are
+	 * distinct.
+	 *
+	 * @return a {@link TrapLocker} for the trap file corresponding to the given jar
+	 *         file.
+	 */
+	public TrapLocker getTrapLockerForJarFile(File jarFile) {
+		return new TrapLocker(jarFile);
+	}
+
+	/**
+	 * <b>CAUTION</b>: to avoid the potential for deadlock between multiple
+	 * concurrent extractor processes,
+	 * only one module {@link TrapLocker} may be open at any time, and the lock must
+	 * be obtained
+	 * <b>after</b> any <b>source</b> file lock. Only one jar or class file or
+	 * module lock may be open at any time.
+	 *
+	 * Trap file extensions (and paths) ensure that source and module file locks are
+	 * distinct.
+	 *
+	 * @return a {@link TrapLocker} for the trap file corresponding to the given
+	 *         module.
+	 */
+	public TrapLocker getTrapLockerForModule(String moduleName) {
+		return new TrapLocker(moduleName);
+	}
+
+	/**
+	 * <b>CAUTION</b>: to avoid the potential for deadlock between multiple
+	 * concurrent extractor processes,
+	 * only one class file {@link TrapLocker} may be open at any time, and the lock
+	 * must be obtained
+	 * <b>after</b> any <b>source</b> file lock. Only one jar or class file lock may
+	 * be open at any time.
+	 *
+	 * Trap file extensions (and paths) ensure that source and class file locks are
+	 * distinct.
+	 *
+	 * @return a {@link TrapLocker} for the trap file corresponding to the given
+	 *         class symbol.
+	 */
+	public TrapLocker getTrapLockerForDecl(KaSymbol sym, String signature, boolean fromSource) {
+		return new TrapLocker(sym, signature, fromSource);
+	}
+
+	public class TrapLocker implements AutoCloseable {
+		private final KaSymbol sym;
+		private final File trapFile;
+		// trapFileBase is used when doing lockless TRAP file writing.
+		// It is trapFile without the #metadata.trap.gz suffix.
+		private File trapFileBase = null;
+		private TrapClassVersion trapFileVersion = null;
+		private final String signature;
+
+		private TrapLocker(KaSymbol decl, String signature, boolean fromSource) {
+			this.sym = decl;
+			this.signature = signature;
+			if (sym == null) {
+				log.error("Null symbol passed for Kotlin TRAP locker");
+				trapFile = null;
+			} else {
+				File normalTrapFile = getTrapFileForDecl(sym, signature);
+				// We encode the metadata into the filename, so that the
+				// TRAP filenames for different metadatas don't overlap.
+				if (fromSource)
+					trapFileVersion = new TrapClassVersion(0, 0, 0, "kotlin");
+				else
+					trapFileVersion = TrapClassVersion.fromSymbol(sym, log);
+				String baseName = normalTrapFile.getName().replace(".trap" + compression.getExtension(), "");
+				// If a class has lots of inner classes, then we get lots of files
+				// in a single directory. This makes our directory listings later slow.
+				// To avoid this, rather than using files named .../Foo*, we use .../Foo/Foo*.
+				trapFileBase = new File(new File(normalTrapFile.getParentFile(), baseName), baseName);
+				trapFile = new File(trapFileBase.getPath() + '#' + trapFileVersion.toString() + ".trap"
+						+ compression.getExtension());
+			}
+		}
+
+		private TrapLocker(File jarFile) {
+			sym = null;
+			signature = null;
+			trapFile = getTrapFileForJarFile(jarFile);
+		}
+
+		private TrapLocker(String moduleName) {
+			sym = null;
+			signature = null;
+			trapFile = getTrapFileForModule(moduleName);
+		}
+
+		public TrapFileManager getTrapFileManager() {
+			if (trapFile != null) {
+				return getMembersWriterForDecl(trapFile, trapFileBase, trapFileVersion, sym, signature);
+			} else {
+				return null;
+			}
+		}
+
+		@Override
+		public void close() {
+			if (trapFile != null) {
+				// Now that we have finished writing our TRAP file, we want
+				// to rename and TRAP file that matches our trapFileBase
+				// but doesn't have the latest metadata.
+				// Renaming it to trap-old means that it won't be imported,
+				// but we can still use its presence to avoid future
+				// invocations rewriting it, and it means that the information
+				// is in the TRAP directory if we need it for debugging.
+				if (sym != null) {
+					File trapFileDir = trapFileBase.getParentFile();
+					String trapFileBaseName = trapFileBase.getName();
+
+					List<Pair<File, TrapClassVersion>> pairs = new LinkedList<Pair<File, TrapClassVersion>>();
+					for (File f : FileUtil.list(trapFileDir)) {
+						String name = f.getName();
+						Matcher m = selectClassVersionComponents.matcher(name);
+						if (m.matches()) {
+							if (m.group(1).equals(trapFileBaseName)) {
+								TrapClassVersion v = new TrapClassVersion(Integer.valueOf(m.group(2)),
+										Integer.valueOf(m.group(3)), Long.valueOf(m.group(4)), m.group(5));
+								pairs.add(new Pair<File, TrapClassVersion>(f, v));
+							} else {
+								// Everything in this directory should be for the same TRAP file base
+								log.error("Unexpected sibling " + m.group(1) + " when extracting " + trapFileBaseName);
+							}
+						}
+					}
+					if (pairs.isEmpty()) {
+						log.error("Wrote TRAP file, but no TRAP files exist for " + trapFile.getAbsolutePath());
+					} else {
+						Comparator<Pair<File, TrapClassVersion>> comparator = new Comparator<Pair<File, TrapClassVersion>>() {
+							@Override
+							public int compare(Pair<File, TrapClassVersion> p1, Pair<File, TrapClassVersion> p2) {
+								TrapClassVersion v1 = p1.snd();
+								TrapClassVersion v2 = p2.snd();
+								if (v1.equals(v2)) {
+									return 0;
+								} else if (v1.newerThan(v2)) {
+									return 1;
+								} else {
+									return -1;
+								}
+							}
+						};
+						TrapClassVersion latestVersion = Collections.max(pairs, comparator).snd();
+
+						for (Pair<File, TrapClassVersion> p : pairs) {
+							if (!latestVersion.equals(p.snd())) {
+								File f = p.fst();
+								File fOld = new File(f.getParentFile(),
+										f.getName().replace(".trap" + compression.getExtension(),
+												".trap-old" + compression.getExtension()));
+								// We aren't interested in whether or not this succeeds;
+								// it may fail because a concurrent extractor has already
+								// renamed it.
+								f.renameTo(fOld);
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	/*
+	 * Class version tracking.
+	 */
+
+	private static class TrapClassVersion {
+		private int majorVersion;
+		private int minorVersion;
+		private long lastModified;
+		private String extractorName; // May be null if not given
+
+		public int getMajorVersion() {
+			return majorVersion;
+		}
+
+		public int getMinorVersion() {
+			return minorVersion;
+		}
+
+		public long getLastModified() {
+			return lastModified;
+		}
+
+		public String getExtractorName() {
+			return extractorName;
+		}
+
+		private TrapClassVersion(int majorVersion, int minorVersion, long lastModified, String extractorName) {
+			this.majorVersion = majorVersion;
+			this.minorVersion = minorVersion;
+			this.lastModified = lastModified;
+			this.extractorName = extractorName;
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			if (obj instanceof TrapClassVersion) {
+				TrapClassVersion other = (TrapClassVersion) obj;
+				return majorVersion == other.majorVersion && minorVersion == other.minorVersion
+						&& lastModified == other.lastModified && extractorName.equals(other.extractorName);
+			} else {
+				return false;
+			}
+		}
+
+		@Override
+		public int hashCode() {
+			int hash = 7;
+			hash = 31 * hash + majorVersion;
+			hash = 31 * hash + minorVersion;
+			hash = 31 * hash + (int) lastModified;
+			hash = 31 * hash + (extractorName == null ? 0 : extractorName.hashCode());
+			return hash;
+		}
+
+		private boolean newerThan(TrapClassVersion tcv) {
+			// Classes being compiled from source have major version 0 but should take
+			// precedence
+			// over any classes with the same qualified name loaded from the classpath
+			// in previous or subsequent extractor invocations.
+			if (tcv.majorVersion == 0 && majorVersion != 0)
+				return false;
+			else if (majorVersion == 0 && tcv.majorVersion != 0)
+				return true;
+			// Always consider the Kotlin extractor superior to the Java extractor, because
+			// we may decode and extract
+			// Kotlin metadata that the Java extractor can't understand:
+			if (!Objects.equals(tcv.extractorName, extractorName)) {
+				if (Objects.equals(tcv.extractorName, "kotlin"))
+					return false;
+				if (Objects.equals(extractorName, "kotlin"))
+					return true;
+			}
+			// Otherwise, determine precedence in the following order:
+			// majorVersion, minorVersion, lastModified.
+			return tcv.majorVersion < majorVersion ||
+					(tcv.majorVersion == majorVersion && tcv.minorVersion < minorVersion) ||
+					(tcv.majorVersion == majorVersion && tcv.minorVersion == minorVersion &&
+							tcv.lastModified < lastModified);
+		}
+
+		private static Map<String, Map<String, Long>> jarFileEntryTimeStamps = new HashMap<>();
+
+		private static Map<String, Long> getZipFileEntryTimeStamps(String path, Logger log) {
+			try {
+				Map<String, Long> result = new HashMap<>();
+				ZipFile zf = new ZipFile(path);
+				Enumeration<? extends ZipEntry> entries = zf.entries();
+				while (entries.hasMoreElements()) {
+					ZipEntry ze = entries.nextElement();
+					result.put(ze.getName(), ze.getLastModifiedTime().toMillis());
+				}
+				return result;
+			} catch (IOException e) {
+				log.warn("Failed to get entry timestamps from " + path, e);
+				return null;
+			}
+		}
+
+		private static long getVirtualFileTimeStamp(VirtualFile vf, Logger log) {
+			if (vf.getFileSystem().getProtocol().equals("jar")) {
+				String[] parts = vf.getPath().split("!/");
+				if (parts.length == 2) {
+					String jarFilePath = parts[0];
+					String entryPath = parts[1];
+					if (!jarFileEntryTimeStamps.containsKey(jarFilePath)) {
+						jarFileEntryTimeStamps.put(jarFilePath, getZipFileEntryTimeStamps(jarFilePath, log));
+					}
+					Map<String, Long> entryTimeStamps = jarFileEntryTimeStamps.get(jarFilePath);
+					if (entryTimeStamps != null) {
+						Long entryTimeStamp = entryTimeStamps.get(entryPath);
+						if (entryTimeStamp != null)
+							return entryTimeStamp;
+						else
+							log.warn("Couldn't find timestamp for jar file " + jarFilePath + " entry " + entryPath);
+					}
+				} else {
+					log.warn("Expected JAR-file path " + vf.getPath() + " to have exactly one '!/' separator");
+				}
+			}
+
+			// For all files except for jar files, and a fallback in case of I/O problems
+			// reading a jar file:
+			return vf.getTimeStamp();
+		}
+
+		private static VirtualFile getVirtualFileIfClass(KaSymbol e) {
+			// TODO:
+			return null;
+			// if (e instanceof IrClass)
+			// return getIrClassVirtualFile((IrClass) e);
+			// else
+			// return null;
+		}
+
+		private static TrapClassVersion fromSymbol(KaSymbol sym, Logger log) {
+			VirtualFile vf = getVirtualFileIfClass(sym);
+      /* OLD: KE1
+			if (vf == null && sym instanceof IrDeclaration)
+				vf = getVirtualFileIfClass(((IrDeclaration) sym).getParent());
+      */
+			if (vf == null)
+				return new TrapClassVersion(-1, 0, 0, null);
+
+			final int[] versionStore = new int[1];
+
+			try {
+				// Opcodes has fields called ASM4, ASM5, ...
+				// We want to use the latest one that there is.
+				Field asmField = null;
+				int asmNum = -1;
+				for (Field f : Opcodes.class.getDeclaredFields()) {
+					String name = f.getName();
+					if (name.startsWith("ASM")) {
+						try {
+							int i = Integer.parseInt(name.substring(3));
+							if (i > asmNum) {
+								asmNum = i;
+								asmField = f;
+							}
+						} catch (NumberFormatException ex) {
+							// Do nothing; this field doesn't have a name of the right format
+						}
+					}
+				}
+				int asm = asmField.getInt(null);
+				ClassVisitor versionGetter = new ClassVisitor(asm) {
+					public void visit(int version, int access, java.lang.String name, java.lang.String signature,
+							java.lang.String superName, java.lang.String[] interfaces) {
+						versionStore[0] = version;
+					}
+				};
+				(new ClassReader(vf.contentsToByteArray())).accept(versionGetter,
+						ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES);
+
+				return new TrapClassVersion(versionStore[0] & 0xffff, versionStore[0] >> 16,
+						getVirtualFileTimeStamp(vf, log), "kotlin");
+			} catch (IllegalAccessException e) {
+				log.warn("Failed to read class file version information", e);
+				return new TrapClassVersion(-1, 0, 0, null);
+			} catch (IOException e) {
+				log.warn("Failed to read class file version information", e);
+				return new TrapClassVersion(-1, 0, 0, null);
+			}
+		}
+
+		private boolean isValid() {
+			return majorVersion >= 0 && minorVersion >= 0;
+		}
+
+		@Override
+		public String toString() {
+			return majorVersion + "." + minorVersion + "-" + lastModified + "-" + extractorName;
+		}
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/extractor/java/PopulateFile.java

@@ -0,0 +1,152 @@
+package com.semmle.extractor.java;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.github.codeql.Label;
+import com.github.codeql.DbFile;
+import com.github.codeql.TrapWriter;
+import com.github.codeql.KotlinExtractorDbSchemeKt;
+import com.semmle.util.exception.CatastrophicError;
+import com.semmle.util.files.FileUtil;
+import com.semmle.util.trap.pathtransformers.PathTransformer;
+import kotlin.Unit;
+
+public class PopulateFile {
+
+  private TrapWriter tw;
+  private PathTransformer transformer;
+  public PopulateFile(TrapWriter tw) {
+    this.tw = tw;
+    this.transformer = PathTransformer.std();
+  }
+
+	private static final String[] keyReplacementMap = new String[127];
+	static {
+		keyReplacementMap['&'] = "&";
+		keyReplacementMap['{'] = "{";
+		keyReplacementMap['}'] = "}";
+		keyReplacementMap['"'] = """;
+		keyReplacementMap['@'] = "@";
+		keyReplacementMap['#'] = "#";
+	}
+
+	/**
+	 * Escape a string for use in a TRAP key, by replacing special characters with HTML entities.
+	 * <p>
+	 * The given string cannot contain any sub-keys, as the delimiters <code>{</code> and <code>}</code>
+	 * are escaped.
+	 * <p>
+	 * To construct a key containing both sub-keys and arbitrary input data, escape the individual parts of
+	 * the key rather than the key as a whole, for example:
+	 * <pre>
+	 * "foo;{" + label.toString() + "};" + escapeKey(data)
+	 * </pre>
+	 */
+	public static String escapeKey(String s) {
+		StringBuilder sb = null;
+		int lastIndex = 0;
+		for (int i = 0; i < s.length(); ++i) {
+			char ch = s.charAt(i);
+			switch (ch) {
+			case '&':
+			case '{':
+			case '}':
+			case '"':
+			case '@':
+			case '#':
+				if (sb == null) {
+					sb = new StringBuilder();
+				}
+				sb.append(s, lastIndex, i);
+				sb.append(keyReplacementMap[ch]);
+				lastIndex = i + 1;
+				break;
+			}
+		}
+		if (sb != null) {
+			sb.append(s, lastIndex, s.length());
+			return sb.toString();
+		} else {
+			return s;
+		}
+	}
+
+    public Label populateFile(File absoluteFile) {
+        return getFileLabel(absoluteFile, true);
+    }
+
+    public Label<DbFile> getFileLabel(File absoluteFile, boolean populateTables) {
+        String databasePath = transformer.fileAsDatabaseString(absoluteFile);
+        Label result = tw.<DbFile>getLabelFor("@\"" + escapeKey(databasePath) + ";sourcefile" + "\"", label -> {
+            if(populateTables) {
+                KotlinExtractorDbSchemeKt.writeFiles(tw, label, databasePath);
+                populateParents(new File(databasePath), label);
+            }
+            return Unit.INSTANCE;
+        });
+        return result;
+    }
+
+	private Label addFolderTuple(String databasePath) {
+		Label result = tw.getLabelFor("@\"" + escapeKey(databasePath) + ";folder" + "\"");
+		KotlinExtractorDbSchemeKt.writeFolders(tw, result, databasePath);
+		return result;
+	}
+
+	/**
+	 * Populate the parents of an already-normalised file. The path transformers
+	 * and canonicalisation of {@link PathTransformer#fileAsDatabaseString(File)} will not be
+	 * re-applied to this, so it should only be called after proper normalisation
+	 * has happened. It will fill in all parent folders in the current TRAP file.
+	 */
+	private void populateParents(File normalisedFile, Label label) {
+		File parent = normalisedFile.getParentFile();
+		if (parent == null) return;
+
+    Label parentLabel = addFolderTuple(FileUtil.normalisePath(parent.getPath()));
+    populateParents(parent, parentLabel);
+		KotlinExtractorDbSchemeKt.writeContainerparent(tw, parentLabel, label);
+	}
+
+    public Label relativeFileId(File jarFile, String pathWithinJar) {
+        return getFileInJarLabel(jarFile, pathWithinJar, true);
+    }
+
+	public Label<DbFile> getFileInJarLabel(File jarFile, String pathWithinJar, boolean populateTables) {
+		if (pathWithinJar.contains("\\"))
+			throw new CatastrophicError("Invalid jar path: '" + pathWithinJar + "' should not contain '\\'.");
+
+        String databasePath = transformer.fileAsDatabaseString(jarFile);
+        if(!populateTables)
+            return tw.getLabelFor("@\"" + databasePath + "/" + pathWithinJar + ";jarFile\"");
+
+		Label jarFileId = this.populateFile(jarFile);
+        Label jarFileLocation = tw.getLocation(jarFileId, 0, 0, 0, 0);
+        KotlinExtractorDbSchemeKt.writeHasLocation(tw, jarFileId, jarFileLocation);
+
+		StringBuilder fullName = new StringBuilder(databasePath);
+		String[] split = pathWithinJar.split("/");
+		Label current = jarFileId;
+		for (int i = 0; i < split.length; i++) {
+			String shortName = split[i];
+
+			fullName.append("/");
+			fullName.append(shortName);
+			Label fileId = tw.getLabelFor("@\"" + fullName + ";jarFile" + "\"");
+
+			boolean file = i == split.length - 1;
+			if (file) {
+				KotlinExtractorDbSchemeKt.writeFiles(tw, fileId, fullName.toString());
+			} else {
+				KotlinExtractorDbSchemeKt.writeFolders(tw, fileId, fullName.toString());
+			}
+			KotlinExtractorDbSchemeKt.writeContainerparent(tw, current, fileId);
+			current = fileId;
+		}
+
+		return current;
+	}
+
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/array/ArrayUtil.java

@@ -0,0 +1,246 @@
+package com.semmle.util.array;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+import com.semmle.util.basic.ObjectUtil;
+
+/**
+ * Convenience methods for manipulating arrays.
+ */
+public class ArrayUtil
+{
+	
+	/**
+	 * A number slightly smaller than the maximum length of an array on most vms. 
+	 * This matches the constant in ArrayList.
+	 */
+	public static final int MAX_ARRAY_LENGTH = Integer.MAX_VALUE - 8;
+
+	/**
+	 * Comparator for primitive int values.
+	 */
+	public static interface IntComparator
+	{
+		/**
+		 * Compare ints {@code a} and {@code b}, returning a negative value if {@code a} is 'less' than
+		 * {@code b}, zero if they are equal, otherwise a positive value.
+		 */
+		public int compare (int a, int b);
+	}
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no such element.
+	 */
+	public static int findFirst(boolean[] array, boolean value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (value == array[i])
+				return i;
+		}
+		return -1;
+	}
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no such element.
+	 */
+	public static int findFirst(byte[] array, byte value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (value == array[i])
+				return i;
+		}
+		return -1;
+	}
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no such element.
+	 */
+	public static int findFirst(char[] array, char value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (value == array[i])
+				return i;
+		}
+		return -1;
+	}
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no such element.
+	 */
+	public static int findFirst(double[] array, double value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (value == array[i])
+				return i;
+		}
+		return -1;
+	}
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no such element.
+	 */
+	public static int findFirst(float[] array, float value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (value == array[i])
+				return i;
+		}
+		return -1;
+	}
+
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no such element.
+	 */
+	public static int findFirst(int[] array, int value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (value == array[i])
+				return i;
+		}
+		return -1;
+	}
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no element for which {@code value.equals(element)} is true.
+	 *
+	 * @see #findFirstSame(Object[], Object)
+	 */
+	public static <T> int findFirst(T[] array, T value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (ObjectUtil.equals(value, array[i])) {
+				return i;
+			}
+		}
+		return -1;
+	}
+
+	/**
+	 * Find the index of the first occurrence of the given {@code value} in the given {@code array},
+	 * returning -1 if there is no element for which {@code value == element}.
+	 *
+	 * @see #findFirstSame(Object[], Object)
+	 */
+	public static <T> int findFirstSame(T[] array, T value)
+	{
+		for(int i=0; i<array.length; ++i) {
+			if (value == array[i])
+				return i;
+		}
+		return -1;
+	}
+
+	/**
+	 * Query whether the given {@code array} contains any element equal to the given {@code element}.
+	 */
+	public static boolean contains (int element, int ... array)
+	{
+		return findFirst(array, element) != -1;
+	}
+
+	
+
+	/**
+	 * Query whether the given {@code array} contains any element equal to the given {@code element}.
+	 */
+	@SafeVarargs
+	public static <T> boolean contains (T element, T ... array)
+	{
+		return findFirst(array, element) != -1;
+	}
+
+	/**
+	 * Construct a new array with length increased by one, containing all elements of a given array
+	 * followed by an additional element.
+	 */
+	public static <T> T[] append (T[] array, T element)
+	{
+		array = Arrays.copyOf(array, array.length + 1);
+		array[array.length-1] = element;
+		return array;
+	}
+
+	/**
+	 * Construct a new array containing the concatenation of the elements in a number of arrays.
+	 *
+	 * @param arrays The arrays to concatenate; may be null (in which case the result will be null).
+	 *          Null elements will be treated as empty arrays.
+	 * @return If {@code arrays} is null, a null array, otherwise a newly allocated array containing
+	 *         the elements of every non-null array in {@code arrays} concatenated consecutively.
+	 */
+	public static byte[] concatenate (byte[] ... arrays)
+	{
+		// Quick break-out if arrays is null
+		if (arrays == null) {
+			return null;
+		}
+		// Find the total length that will be required
+		int totalLength = 0;
+		for(byte[] array : arrays) {
+			totalLength += array == null ? 0 : array.length;
+		}
+		// Allocate a new array for the concatenation
+		byte[] concatenation = new byte[totalLength];
+		// Copy each non-null array into the concatenation
+		int offset = 0;
+		for(byte[] array : arrays) {
+			if (array != null) {
+				System.arraycopy(array, 0, concatenation, offset, array.length);
+				offset += array.length;
+			}
+		}
+
+		return concatenation;
+	}
+
+	/** Trivial short-hand for building an array (returns {@code elements} unchanged). */
+	public static <T> T[] toArray (T ... elements)
+	{
+		return elements;
+	}
+	
+	/**
+	 * Swap two elements in an array.
+	 *
+	 * @param array The array containing the elements to be swapped; must be non-null.
+	 * @param index1 The index of the first element to swap; must be in-bounds.
+	 * @param index2 The index of the second element to swap; must be in-bounds.
+	 * @return The given {@code array}.
+	 */
+	public static int[] swap (int[] array, int index1, int index2)
+	{
+		int value     = array[index1];
+		array[index1] = array[index2];
+		array[index2] = value;
+
+		return array;
+	}
+
+	/**
+	 * Returns a fresh Set containing all the elements in the array.
+	 *
+	 * @param <T>
+	 *            the class of the objects in the array
+	 * @param array
+	 *            the array containing the elements
+	 * @return a Set containing all the elements in the array.
+	 */
+	@SafeVarargs
+	public static <T> Set<T> asSet (T ... array)
+	{
+		Set<T> ts = new LinkedHashSet<>();
+		Collections.addAll(ts, array);
+		return ts;
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/basic/ObjectUtil.java

@@ -0,0 +1,73 @@
+package com.semmle.util.basic;
+
+/**
+ * Trivial utility methods.
+ */
+public class ObjectUtil {
+
+	/** Query if {@code object1} and {@code object2} are reference-equal, or both null. */
+	public static boolean isSame (Object object1, Object object2)
+	{
+		return object1 == object2; // Reference equality comparison is deliberate
+	}
+
+	/**
+	 * Query if {@code object1} and {@code object2} are both null, or both non-null and equal
+	 * according to {@link Object#equals(Object)} (applied as {@code object1.equals(object2)}).
+	 */
+	public static boolean equals (Object object1, Object object2)
+	{
+		return object1 == null ? object2 == null : object1.equals(object2);
+	}
+
+	/**
+	 * Query whether {@code object} is equal to any element in {@code objects}, short-circuiting
+	 * the evaluation if possible.
+	 */
+	public static boolean equalsAny (Object object, Object ... objects)
+	{
+		// Quick break-out if there are no objects to be equal to
+		if (objects == null || objects.length == 0) {
+			return false;
+		}
+		// Compare against each object in turn
+		for(Object other : objects) {
+			if (equals(object, other)) {
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	/**
+	 * Return {@code object1.compareTo(object2)}, but handle the case of null input by returning 0 if
+	 * both objects are null, or 1 if only {@code object1} is null (implying that null is always
+	 * 'greater' than non-null).
+	 */
+	public static <T1, T2 extends T1> int compareTo (Comparable<T1> object1, T2 object2)
+	{
+		if (object1 == null) {
+			return object2 == null ? 0 : 1;
+		}
+		return object1.compareTo(object2);
+	}
+
+	/**
+	 * Return {@code value} if non-null, otherwise {@code replacement}.
+	 */
+	public static <T> T replaceNull (T value, T replacement)
+	{
+		return value == null ? replacement : value;
+	}
+	
+	@SafeVarargs
+	public static <T> T nullCoalesce(T... values) {
+		for(T value : values) {
+			if (value != null) {
+				return value;
+			}
+		}
+		return null;
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/concurrent/LockDirectory.java

@@ -0,0 +1,395 @@
+package com.semmle.util.concurrent;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import com.semmle.util.data.StringDigestor;
+import com.semmle.util.exception.CatastrophicError;
+import com.semmle.util.exception.ResourceError;
+import com.semmle.util.files.FileUtil;
+import com.semmle.util.io.WholeIO;
+
+import com.github.codeql.Logger;
+import com.github.codeql.Severity;
+
+/**
+ * Helper class to simplify handling of file-system-based inter-process
+ * locking and mutual exclusion.
+ *
+ * Both files and directories can be locked; locks are provided in the
+ * usual flavours of "shared" and "exclusive", plus a no-op variety to
+ * help unify code -- see the {@link LockingMode} enum.
+ *
+ * Note that each locked file requires one file descriptor to be held open.
+ * It is vital for clients to avoid creating too many locks, and to release
+ * locks when possible.
+ *
+ * The locks obtained by this class are VM-wide, and cannot be used to
+ * ensure mutual exclusion between threads of the same VM. Rather, they
+ * can enforce mutual exclusion between separate VMs trying to acquire
+ * locks for the same paths.
+ */
+public class LockDirectory {
+	private final Logger logger;
+
+	private final File lockDir;
+
+	/**
+	 * An enum describing the possible locking modes.
+	 */
+	public enum LockingMode {
+		/**
+		 * Shared mode: A shared lock can be taken any number of times, but only
+		 * if no exclusive lock is in place.
+		 */
+		Shared(true),
+		/**
+		 * An exclusive lock can only be taken if no other lock is in place; it
+		 * prevents all other locks.
+		 */
+		Exclusive(false),
+		/**
+		 * A dummy mode: Lock and unlock operations are no-ops.
+		 */
+		None(true),
+		;
+
+		private boolean shared;
+
+		private LockingMode(boolean shared) {
+			this.shared = shared;
+		}
+
+		public boolean isShared() { return shared; }
+	}
+
+	/**
+	 * An internal representation of a locked path. Contains some immutable state: The canonical
+	 * path being locked, and the (derived) lock and status files. When the {@link #lock(LockDirectory.LockingMode, String)}
+	 * method is called, a file descriptor to the lock file is opened; {@link #unlock(LockDirectory.LockingMode)} must be
+	 * called to release it when the lock is no longer required.
+	 *
+	 * This class is not thread-safe, but it is expected that its clients ({@link LockDirectory})
+	 * enforce thread-safe access to instances.
+	 */
+	private class LockFile {
+		private final String lockedPath;
+		private final File lockFile;
+		private final File statusFile;
+
+		private LockingMode mode = null;
+		private RandomAccessFile lockStream = null;
+		private FileChannel lockChannel = null;
+		private FileLock lock = null;
+
+		public LockFile(File f) {
+			try {
+				lockedPath = f.getCanonicalPath();
+			} catch (IOException e) {
+				throw new ResourceError("Failed to canonicalise path for locking: " + f, e);
+			}
+			String sha = StringDigestor.digest(lockedPath);
+			lockFile = new File(lockDir, sha);
+			statusFile = new File(lockDir, sha + ".log");
+		}
+
+		/**
+		 * Get the (canonical) path associated with this lock file -- this is the
+		 * path that is being locked.
+		 */
+		public String getLockedPath() {
+			return lockedPath;
+		}
+
+		/**
+		 * Acquire a lock with the given mode. If this method returns normally,
+		 * the lock has been acquired -- an exception is thrown otherwise. This
+		 * method does not block.
+		 *
+		 * If no exception is thrown, a file descriptor is kept open until
+		 * {@link #unlock(LockDirectory.LockingMode)} is called.
+		 * @param mode The desired locking mode. If {@link LockingMode#None}, this
+		 * 	operation is a no-op (and does not in fact open a file descriptor).
+		 * @param message A message to be recorded alongside the lock file. This
+		 * is included in the exception message of other processes using this
+		 * 	infrastructure when the lock acquisition fails.
+		 * @throws CatastrophicError if a lock has already been obtained and not released.
+		 * @throws ResourceError if an exception occurs while obtaining the lock, including
+		 * 	if it cannot be acquired because another process holds it.
+		 */
+		public void lock(LockingMode mode, String message) {
+			if (mode == LockingMode.None) return;
+			if (lock != null)
+				throw new CatastrophicError("Trying to re-lock existing lock for " + lockedPath);
+			this.mode = mode;
+			try {
+				lockStream = new RandomAccessFile(lockFile, "rw");
+				lockChannel = lockStream.getChannel();
+				tryLock(mode);
+				new WholeIO().strictwrite(statusFile, mode + " lock acquired for " + lockedPath + ": " + message);
+			} catch (IOException e) {
+				throw new ResourceError("Failed to obtain lock for " + lockedPath + " at " + lockFile, e);
+			}
+		}
+
+		/**
+		 * Acquire a lock with the given mode. If this method returns normally,
+		 * the lock has been acquired -- an exception is thrown otherwise. This
+		 * method blocks indefinitely while waiting to acquire the lock.
+		 *
+		 * If no exception is thrown, a file descriptor is kept open until
+		 * {@link #unlock(LockDirectory.LockingMode)} is called.
+		 * @param mode The desired locking mode. If {@link LockingMode#None}, this
+		 * 	operation is a no-op (and does not in fact open a file descriptor).
+		 * @param message A message to be recorded alongside the lock file. This
+		 * is included in the exception message of other processes using this
+		 * 	infrastructure when the lock acquisition fails.
+		 * @throws ResourceError if an exception occurs while obtaining the lock,.
+		 */
+		public void blockingLock(LockingMode mode, String message) {
+			if (mode == LockingMode.None) return;
+			if (lock != null)
+				throw new CatastrophicError("Trying to re-lock existing lock for " + lockedPath);
+			this.mode = mode;
+			try {
+				lockStream = new RandomAccessFile(lockFile, "rw");
+				lockChannel = lockStream.getChannel();
+				lock = lockChannel.tryLock(0, Long.MAX_VALUE, mode.isShared());
+				while (lock == null) {
+					ThreadUtil.sleep(500, true);
+					lock = lockChannel.tryLock(0, Long.MAX_VALUE, mode.isShared());
+				}
+				new WholeIO().strictwrite(statusFile, mode + " lock acquired for " + lockedPath + ": " + message);
+			} catch (IOException e) {
+				throw new ResourceError("Failed to obtain lock for " + lockedPath + " at " + lockFile, e);
+			}
+		}
+
+		/**
+		 * Internal helper method: Try to acquire a particular kind of lock, assuming the
+		 * {@link #lockChannel} has been set up. Throws if acquisition fails, rather than
+		 * blocking.
+		 * @param mode The desired lock mode -- exclusive or shared.
+		 * @throws IOException if acquisition of the lock fails for reasons other than
+		 * 	an incompatible lock already being held by another process.
+		 * @throws ResourceError if the lock is already held by another process. The exception
+		 * 	message includes the status string, if it can be determined.
+		 */
+		private void tryLock(LockingMode mode) throws IOException {
+			lock = lockChannel.tryLock(0, Long.MAX_VALUE, mode.isShared());
+			if (lock == null) {
+				String status = new WholeIO().read(statusFile);
+				throw new ResourceError("Failed to acquire " + mode + " lock for " + lockedPath + "." +
+						(status == null ? "" : "\nExisting lock message: " + status));
+			}
+		}
+
+		/**
+		 * Release this lock. This will close the file descriptor opened by {@link #lock(LockDirectory.LockingMode, String)}.
+		 * @param mode A mode, which must match the mode passed into {@link #lock(LockDirectory.LockingMode, String)}
+		 * 	(unless it is {@link LockingMode#None}, in which case the method is a no-op).
+		 * @throws CatastrophicError if the passed mode does not match the one used for locking.
+		 * @throws ResourceError if releasing the lock or clearing up temporary files fails.
+		 */
+		public void unlock(LockingMode mode) {
+			if (mode == LockingMode.None)
+				return;
+			if (mode != this.mode)
+				throw new CatastrophicError("Attempting to unlock " + lockedPath + " with incompatible mode: " +
+						this.mode + " lock was obtained, but " + mode + " lock is being released.");
+			release(mode);
+		}
+
+		private void release(LockingMode mode) {
+			try {
+				if (lock != null)
+					try {
+						// On Windows, the lockChannel/lockStream prevents the lockFile from being
+						// deleted. The statusFile should only be written after the lock is held,
+						// so deleting it before releasing the lock is not expected to fail if the
+						// lock is exclusive.
+						// Deleting the lock file may fail, if another process just acquires it
+						// after we release it.
+						try {
+							if (statusFile.exists() && !statusFile.delete()) {
+									if (!mode.isShared()) throw new ResourceError("Could not clear status file " + statusFile);
+							}
+						} finally {
+							lock.release();
+							FileUtil.close(lockStream);
+							FileUtil.close(lockChannel);
+							if (!lockFile.delete())
+								logger.error("Could not clear lock file " + lockFile + " (it might have been locked by another process).");
+						}
+					} catch (IOException e) {
+						throw new ResourceError("Couldn't release lock for " + lockedPath, e);
+					}
+			} finally {
+				mode = null;
+				lockStream = null;
+				lockChannel = null;
+				lock = null;
+			}
+		}
+	}
+
+	private static final Map<File, LockDirectory> instances = new LinkedHashMap<File, LockDirectory>();
+
+	/**
+	 * Obtain the {@link LockDirectory} instance for a given lock directory. The directory
+	 * in question will be created if it doesn't exist.
+	 * @param lockDir A directory -- must be writable, and will be created if it doesn't
+	 * 	already exist.
+	 * @return The {@link LockDirectory} instance responsible for the specified lock directory.
+	 * @throws ResourceError if the directory cannot be created, exists as a non-directory
+	 * 	or cannot be canonicalised.
+	 */
+	public static synchronized LockDirectory instance(File lockDir) {
+		return instance(lockDir, null);
+	}
+
+	/**
+	 * See {@link #instance(File)}.
+	 * Use this method only if log output should be directed to a custom {@link Logger}.
+	 */
+	public static synchronized LockDirectory instance(File lockDir, Logger logger) {
+		// First try to create the directory -- canonicalisation will fail if it doesn't exist.
+		try {
+			FileUtil.mkdirs(lockDir);
+		} catch(ResourceError e) {
+			throw new ResourceError("Couldn't ensure lock directory " + lockDir + " exists.", e);
+		}
+
+		// Canonicalise.
+		try {
+			lockDir = lockDir.getCanonicalFile();
+		} catch (IOException e) {
+			throw new ResourceError("Couldn't canonicalise requested lock directory " + lockDir, e);
+		}
+
+		// Find and return the right instance.
+		LockDirectory instance = instances.get(lockDir);
+		if (instance == null) {
+			instance = new LockDirectory(lockDir, logger);
+			instances.put(lockDir, instance);
+		}
+		return instance;
+	}
+
+	/**
+	 * A map from canonical locked paths to the associated {@link LockFile} instances.
+	 */
+	private final Map<String, LockFile> locks = new LinkedHashMap<String, LockFile>();
+
+	/**
+	 * Create a new instance of {@link LockDirectory}, holding all locks in the
+	 * specified log directory.
+	 * @param lockDir A writable directory in which locks will be stored.
+	 * @param logger The {@link Logger} to use, if non-null.
+	 */
+	private LockDirectory(File lockDir, Logger logger) {
+		this.lockDir = lockDir;
+		this.logger = logger;
+	}
+
+	/**
+	 * Acquire a lock of the specified kind for the path represented by the given file.
+	 * The file should exist, and its path should be canonicalisable.
+	 *
+	 * Calling this method keeps one file descriptor open
+	 * @param mode The desired locking mode. If {@link LockingMode#None} is passed, this is a no-op,
+	 * 	otherwise it determines whether a shared or exclusive lock is acquired.
+	 * @param f The path that should be locked -- does not need to be writable, and will not
+	 * 	be opened.
+	 * @param message A message describing the purpose of the lock acquisition. This is
+	 * 	potentially displayed when other processes fail to acquire the lock for the given
+	 * 	path.
+	 * @throws CatastrophicError if an attempt is made to lock an already locked path.
+	 */
+	public synchronized void lock(LockingMode mode, File f, String message) {
+		if (mode == LockingMode.None) return;
+		LockFile lock = new LockFile(f);
+		if (locks.containsKey(lock.getLockedPath()))
+				throw new CatastrophicError("Trying to lock already locked path " + lock.getLockedPath() + ".");
+		lock.lock(mode, message);
+		locks.put(lock.getLockedPath(), lock);
+	}
+
+	/**
+	 * Acquire a lock of the specified kind for the path represented by the given file.
+	 * The file should exist, and its path should be canonicalisable. This method waits
+	 * indefinitely for the lock to become available. There is no ordering on processes
+	 * that are waiting to acquire the lock in this manner.
+	 *
+	 * Calling this method keeps one file descriptor open
+	 * @param mode The desired locking mode. If {@link LockingMode#None} is passed, this is a no-op,
+	 * 	otherwise it determines whether a shared or exclusive lock is acquired.
+	 * @param f The path that should be locked -- does not need to be writable, and will not
+	 * 	be opened.
+	 * @param message A message describing the purpose of the lock acquisition. This is
+	 * 	potentially displayed when other processes fail to acquire the lock for the given
+	 * 	path.
+	 */
+	public synchronized void blockingLock(LockingMode mode, File f, String message) {
+		if (mode == LockingMode.None) return;
+		LockFile lock = new LockFile(f);
+		if (locks.containsKey(lock.getLockedPath()))
+			throw new CatastrophicError("Trying to lock already locked path " + lock.getLockedPath() + ".");
+		lock.blockingLock(mode, message);
+		locks.put(lock.getLockedPath(), lock);
+	}
+
+	/**
+	 * Release a lock held on a particular path.
+	 *
+	 * This method closes the file descriptor associated with the lock, freeing related
+	 * resources.
+	 * @param mode the mode of the lock. If it equals {@link LockingMode#None}, this is a no-op; otherwise
+	 * 	it is expected to match the mode passed to the corresponding {@link #lock(LockingMode, File, String)}
+	 * 	call.
+	 * @param f The path which should be unlocked. As with {@link #lock(LockingMode, File, String)}, it is
+	 *	expected to exist and be canonicalisable. It also must be currently locked.
+	 * @throws CatastrophicError on API contract violation: The path isn't currently locked, or the
+	 * 	mode doesn't correspond to the mode specified when it was locked.
+	 * @throws ResourceError if something goes wrong while releasing resources.
+	 */
+	public synchronized void unlock(LockingMode mode, File f) {
+		if (!maybeUnlock(mode, f))
+			throw new CatastrophicError("Trying to unlock " + new LockFile(f).getLockedPath() + ", but it is not locked.");
+	}
+
+	/**
+	 * Release a lock that may be held on a particular path.
+	 *
+	 * This method closes the file descriptor associated with the lock, freeing related
+	 * resources. Unlike {@link #unlock(LockingMode, File)}, this method will not throw
+	 * if the specified {@link File} is not locked, making it more suitable for post-exception
+	 * cleanup -- <code>false</code> will be returned in that case.
+	 * @param mode the mode of the lock. If it equals {@link LockingMode#None}, this is a no-op; otherwise
+	 * 	it is expected to match the mode passed to the corresponding {@link #lock(LockingMode, File, String)}
+	 * 	call.
+	 * @param f The path which should be unlocked. As with {@link #lock(LockingMode, File, String)}, it is
+	 *	expected to exist and be canonicalisable.
+	 * @return <code>true</code> if <code>mode == LockingMode.None</code>, or the unlock operation completed
+	 * 		successfully; <code>false</code> if the path <code>f</code> isn't currently locked.
+	 * @throws ResourceError if something goes wrong while releasing resources.
+	 */
+	public synchronized boolean maybeUnlock(LockingMode mode, File f) {
+		if (mode == LockingMode.None) return true;
+		// New instance constructed just to share the logic of computing the canonical path.
+		LockFile key = new LockFile(f);
+		LockFile existing = locks.get(key.getLockedPath());
+		if (existing == null)
+			return false;
+		locks.remove(key.getLockedPath());
+		existing.unlock(mode);
+		return true;
+	}
+
+	public File getDir(){ return lockDir; }
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/concurrent/ThreadUtil.java

@@ -0,0 +1,43 @@
+package com.semmle.util.concurrent;
+
+import com.semmle.util.exception.CatastrophicError;
+import com.semmle.util.exception.Exceptions;
+
+
+/**
+ * Utility methods related to Threads.
+ */
+public enum ThreadUtil
+{
+	/** Singleton instance of {@link ThreadUtil}. */
+	SINGLETON;
+
+	/**
+	 * Sleep for {@code millis} milliseconds.
+	 * <p>
+	 * Unlike {@link Thread#sleep(long)} (which is wrapped), this method does not throw an
+	 * {@link InterruptedException}, rather in the event of interruption it either throws an
+	 * {@link CatastrophicError} (if {@code allowInterrupt} is false), or accepts the interruption and
+	 * returns false.
+	 * </p>
+	 *
+	 * @return true if a sleep of {@code millis} milliseconds was performed without interruption, or
+	 *         false if an interruption occurred.
+	 */
+	public static boolean sleep(long millis, boolean allowInterrupt)
+	{
+		try {
+			Thread.sleep(millis);
+		}
+		catch (InterruptedException ie) {
+			if (allowInterrupt) {
+				Exceptions.ignore(ie, "explicitly permitted interruption");
+				return false;
+			}
+			else {
+				throw new CatastrophicError("Interrupted", ie);
+			}
+		}
+		return true;
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/data/IntRef.java

@@ -0,0 +1,19 @@
+package com.semmle.util.data;
+
+/**
+ * A mutable reference to a primitive int. Specialised to avoid
+ * boxing.
+ *
+ */
+public class IntRef {
+	private int value;
+	
+	public IntRef(int value) {
+		this.value = value;
+	}
+	
+	public int get() { return value; }
+	public void set(int value) { this.value = value; }
+	public void inc() { value++; }
+	public void add(int val) { value += val; };
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/data/Pair.java

@@ -0,0 +1,62 @@
+package com.semmle.util.data;
+
+
+/**
+ * An (immutable) ordered pair of values.
+ * <p>
+ * Pairs are compared with structural equality: <code>(x,y) = (x', y')</code> iff <code>x=x'</code>
+ * and <code>y=y'</code>.
+ * </p>
+ *
+ * @param <X> the type of the first component of the pair
+ * @param <Y> the type of the second component of the pair
+ */
+public class Pair<X,Y> extends Tuple2<X, Y>
+{
+	private static final long serialVersionUID = -2871892357006076659L;
+	
+	/*
+	 * Constructor and factory
+	 */
+
+
+	/**
+	 * Create a new pair of values
+	 * @param x the first component of the pair
+	 * @param y the second component of the pair
+	 */
+	public Pair(X x, Y y) {
+		super(x, y);
+	}
+
+	/**
+	 * Create a new pair of values. This behaves identically
+	 * to the constructor, but benefits from type inference
+	 * @param x the first component of the pair
+	 * @param y the second component of the pair
+	 */
+	public static <X,Y> Pair<X,Y> make(X x, Y y) {
+		return new Pair<X,Y>(x, y);
+	}
+
+	/*
+	 * Getters
+	 */
+
+	/**
+	 * Get the first component of this pair
+	 * @return the first component of the pair
+	 */
+	public X fst() {
+		return value0();
+	}
+
+	/**
+	 * Get the second component of this pair
+	 * @return the second component of the pair
+	 */
+	public Y snd() {
+		return value1();
+	}
+
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/data/StringDigestor.java

@@ -0,0 +1,173 @@
+package com.semmle.util.data;
+
+import java.nio.charset.Charset;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import com.semmle.util.exception.CatastrophicError;
+
+/**
+ * Encapsulate the creation of message digests from strings.
+ * 
+ * <p>
+ * This class acts as a (partial) output stream, until the <code>getDigest()</code> method is
+ * called. After this the class can no longer be used, except to repeatedly call
+ * {@link #getDigest()}.
+ * 
+ * <p>
+ * UTF-8 is used internally as the {@link Charset} for this class when converting Strings to bytes.
+ */
+public class StringDigestor {
+	private static final Charset UTF8 = Charset.forName("UTF-8");
+	private static final String NULL_STRING = "<null>";
+	private static final int CHUNK_SIZE = 32;
+
+	private MessageDigest digest;
+	private byte[] digestBytes;
+	private final byte[] buf = new byte[CHUNK_SIZE * 3]; // A Java char becomes at most 3 bytes of UTF-8
+
+	/**
+	 * Create a StringDigestor using SHA-1, ready to accept data
+	 */
+	public StringDigestor() {
+		this("SHA-1");
+	}
+
+	/**
+	 * @param digestAlgorithm the algorithm to use in the internal {@link MessageDigest}.
+	 */
+	public StringDigestor(String digestAlgorithm) {
+		try {
+			digest = MessageDigest.getInstance(digestAlgorithm);
+		} catch (NoSuchAlgorithmException e) {
+			throw new CatastrophicError("StringDigestor failed to find the required digest algorithm: " + digestAlgorithm, e);
+		}
+	}
+
+	public void reset() {
+		if (digestBytes == null) throw new CatastrophicError("API violation: Digestor is not finished.");
+		digest.reset();
+		digestBytes = null;
+	}
+
+	/**
+	 * Write an object into this digestor. This converts the object to a
+	 * string using toString(), writes the length, and then writes the
+	 * string itself.
+	 */
+	public StringDigestor write(Object toAppend) {
+		String str;
+		if (toAppend == null) {
+			str = NULL_STRING;
+		} else {
+			str = toAppend.toString();
+		}
+		writeBinaryInt(str.length());
+		writeNoLength(str);
+		return this;
+	}
+
+	/**
+	 * Write the given string without prefixing it by its length. 
+	 */
+	public StringDigestor writeNoLength(Object toAppend) {
+		String s = toAppend.toString();
+		int len = s.length();
+		int i = 0;
+		while(i + CHUNK_SIZE < len) {
+			i = writeUTF8(s, i, i + CHUNK_SIZE);
+		}
+		writeUTF8(s, i, len);
+		return this;
+	}
+
+	private int writeUTF8(String s, int begin, int end) {
+		if (digestBytes != null) throw new CatastrophicError("API violation: Digestor is finished.");
+		byte[] buf = this.buf;
+		int len = 0;
+		for(int i = begin; i < end; ++i) {
+			int c = s.charAt(i);
+			if (c <= 0x7f) {
+				buf[len++] = (byte)c;
+			} else if (c <= 0x7ff) {
+				buf[len] = (byte)(0xc0 | (c >> 6));
+				buf[len+1] = (byte)(0x80 | (c & 0x3f));
+				len += 2;
+			} else if (c < 0xd800 || c > 0xdfff) {
+				buf[len] = (byte)(0xe0 | (c >> 12));
+				buf[len+1] = (byte)(0x80 | ((c >> 6) & 0x3f));
+				buf[len+2] = (byte)(0x80 | (c & 0x3f));
+				len += 3;
+			} else if (i + 1 < end) {
+				int c2 = s.charAt(i + 1);
+				if (c > 0xdbff || c2 < 0xdc00 || c2 > 0xdfff) {
+					// Invalid UTF-16
+				} else {
+					c = 0x10000 + ((c - 0xd800) << 10) + (c2 - 0xdc00);
+					buf[len] = (byte)(0xf0 | (c >> 18));
+					buf[len+1] = (byte)(0x80 | ((c >> 12) & 0x3f));
+					buf[len+2] = (byte)(0x80 | ((c >> 6) & 0x3f));
+					buf[len+3] = (byte)(0x80 | (c & 0x3f));
+					len += 4;
+					++i;
+				}
+			} else {
+				--end;
+				break;
+			}
+		}
+		digest.update(buf, 0, len);
+		return end;
+	}
+
+	/**
+	 * Write an array of raw bytes to the digestor. This appends the contents
+	 * of the array to the accumulated data used for the digest.
+	 */
+	public StringDigestor writeBytes(byte[] data) {
+		if (digestBytes != null) throw new CatastrophicError("API violation: Digestor is finished.");
+		digest.update(data);
+		return this;
+	}
+
+	/**
+	 * Return the hex-encoded digest as a {@link String}.
+	 * 
+	 * Get the digest from the data previously appended using <code>write(Object)</code>.
+	 * After this is called, the instance's {@link #write(Object)} and {@link #writeBytes(byte[])}
+	 * methods may no longer be used.
+	 */
+	public String getDigest() {
+		if (digestBytes == null) {
+			digestBytes = digest.digest();
+		}
+		return StringUtil.toHex(digestBytes);
+	}
+
+	public static String digest(Object o) {
+		StringDigestor digestor = new StringDigestor();
+		digestor.writeNoLength(o);
+		return digestor.getDigest();
+	}
+
+	/** Compute a git-style SHA for the given string. */
+	public static String gitBlobSha(String content) {
+		byte[] bytes = content.getBytes(UTF8);
+		return digest("blob " + bytes.length + "\0" + content);
+	}
+
+	/**
+	 * Convert an int to a byte[4] using its little-endian 32bit representation, and append the
+	 * resulting bytes to the accumulated data used for the digest.
+	 */
+	public StringDigestor writeBinaryInt(int i) {
+		if (digestBytes != null) throw new CatastrophicError("API violation: Digestor is finished.");
+		byte[] buf = this.buf;
+		buf[0] = (byte)(i & 0xff);
+		buf[1] = (byte)((i >>> 8) & 0xff);
+		buf[2] = (byte)((i >>> 16) & 0xff);
+		buf[3] = (byte)((i >>> 24) & 0xff);
+		digest.update(buf, 0, 4);
+		return this;
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/data/Tuple1.java

@@ -0,0 +1,106 @@
+package com.semmle.util.data;
+
+import java.io.Serializable;
+
+
+/**
+ * Tuple of one typed element.
+ * <p>
+ * Note that this is a sub-class of {@link TupleN} and a super-class of {@link Tuple2},
+ * {@link Tuple3}, and any subsequent extensions in a similar vein.
+ * </p>
+ */
+public class Tuple1 <Type0> extends TupleN
+{
+	/**
+	 * Serializable variant of {@link Tuple1}.
+	 */
+	public static class SerializableTuple1<T0 extends Serializable>
+		extends Tuple1<T0> implements Serializable {
+
+		private static final long serialVersionUID = -7989122667707773448L;
+
+		public SerializableTuple1() {
+		}
+
+		public SerializableTuple1(T0 t0) {
+			super(t0);
+		}
+	}
+
+	private static final long serialVersionUID = -4317563803154647477L;
+
+	/** The single contained value. */
+	protected Type0 _value0;
+
+
+	/** Construct a new {@link Tuple1} with a null value. */
+	public Tuple1 () {}
+
+	/** Construct a new {@link Tuple1} with the given value. */
+	public Tuple1 (Type0 value0)
+	{
+		_value0 = value0;
+	}
+
+	/** Construct a new {@link Tuple1} with the given value. */
+	public static <Type0> Tuple1<Type0> make(Type0 value0)
+	{
+		return new Tuple1<Type0>(value0);
+	}
+
+	/**
+	 * Get the value contained by this {@link Tuple1}.
+	 */
+	public final Type0 value0 ()
+	{
+		return _value0;
+	}
+
+	@Override
+	protected Object value_ (int n)
+	{
+		return _value0;
+	}
+
+	/**
+	 * Return the number of elements in this {@link Tuple1}.
+	 * <p>
+	 * Sub-classes shall override this method to increase its value accordingly.
+	 * </p>
+	 */
+	@Override
+	public int size ()
+	{
+		return 1;
+	}
+
+	/**
+	 * Return a plain string representation of the contained value (where null is represented by the
+	 * empty string).
+	 * <p>
+	 * Sub-classes shall implement a comma-separated concatenation.
+	 * </p>
+	 */
+	@Override
+	public String toPlainString ()
+	{
+		return _value0 == null ? "" : _value0.toString();
+	}
+
+	@Override
+	public int hashCode ()
+	{
+		final int prime = 31;
+		int result = super.hashCode();
+		result = prime * result + ((_value0 == null) ? 0 : _value0.hashCode());
+		return result;
+	}
+
+	@Override
+	public boolean equals (Object obj)
+	{
+		return obj == this || (super.equals(obj) && equal(((Tuple1<?>)obj)._value0, _value0));
+	}
+
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/data/Tuple2.java

@@ -0,0 +1,93 @@
+package com.semmle.util.data;
+
+import java.io.Serializable;
+
+
+/**
+ * Tuple of two typed elements.
+ * <p>
+ * Note that this is an extension of {@link Tuple1} and a super-class of {@link Tuple3} (and any
+ * subsequent additions).
+ * </p>
+ */
+public class Tuple2 <Type0, Type1> extends Tuple1<Type0>
+{
+	/**
+	 * Serializable variant of {@link Tuple2}.
+	 */
+	public static class SerializableTuple2<T0 extends Serializable, T1 extends Serializable>
+		extends Tuple2<T0, T1> implements Serializable {
+
+		private static final long serialVersionUID = 1624467154864321244L;
+
+		public SerializableTuple2() {
+		}
+
+		public SerializableTuple2(T0 t0, T1 t1) {
+			super(t0, t1);
+		}
+	}
+
+	private static final long serialVersionUID = -400406676673562583L;
+
+	/** The additional element contained by this {@link Tuple2}. */
+	protected Type1 _value1;
+
+	/** Construct a new {@link Tuple2} with null values. */
+	public Tuple2 () {}
+
+	/** Construct a new {@link Tuple2} with the given values. */
+	public Tuple2 (Type0 value0, Type1 value1)
+	{
+		super(value0);
+		_value1 = value1;
+	}
+
+	/** Construct a new {@link Tuple2} with the given value. */
+	public static <Type1, Type2> Tuple2<Type1, Type2> make(Type1 value0, Type2 value1)
+	{
+		return new Tuple2<Type1,Type2>(value0, value1);
+	}
+
+	/**
+	 * Get the second value in this {@link Tuple2}.
+	 */
+	public final Type1 value1 ()
+	{
+		return _value1;
+	}
+
+	@Override
+	protected Object value_ (int n)
+	{
+		return n == 2 ? _value1 : super.value_(n);
+	}
+
+	@Override
+	public int size ()
+	{
+		return 2;
+	}
+
+	@Override
+	public String toPlainString ()
+	{
+		return super.toPlainString() + ", " + (_value1 == null ? "" : _value1.toString());
+	}
+
+	@Override
+	public int hashCode ()
+	{
+		final int prime = 31;
+		int result = super.hashCode();
+		result = prime * result + ((_value1 == null) ? 0 : _value1.hashCode());
+		return result;
+	}
+
+	@Override
+	public boolean equals (Object obj)
+	{
+		return obj == this || (super.equals(obj) && equal(((Tuple2<?,?>)obj)._value1, _value1));
+	}
+
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/data/TupleN.java

@@ -0,0 +1,85 @@
+package com.semmle.util.data;
+
+import java.io.Serializable;
+
+/**
+ * Untyped base-class for the generic {@link Tuple1}, {@link Tuple2}, ... <i>etc.</i>
+ * <p>
+ * This class also functions as a zero-element tuple.
+ * </p>
+ */
+public class TupleN implements Serializable
+{
+	private static final long serialVersionUID = -1799116497122427806L;
+
+	/**
+	 * Get the n'th value contained by this {@link TupleN}.
+	 *
+	 * @param n The zero-based index of the value to be returned.
+	 * @return The n'th value, or null if n is out of range.
+	 */
+	public final Object value (int n)
+	{
+		return n < 0 || n > size() ? null : value_(n);
+	}
+
+	/** Internal method for obtaining the n'th value (n is guaranteed to be in-range). */
+	protected Object value_ (int n)
+	{
+		return null;
+	}
+
+	/**
+	 * Get the number of values contained by this {@link TupleN}.
+	 */
+	public int size ()
+	{
+		return 0;
+	}
+
+	/**
+	 * Return a plain string representation of the contained value (where null is represented by the
+	 * empty string).
+	 * <p>
+	 * Sub-classes shall implement a comma-separated concatenation.
+	 * </p>
+	 */
+	public String toPlainString ()
+	{
+		return "";
+	}
+
+	/**
+	 * Get a parenthesized, comma-separated string representing the values contained by this
+	 * {@link TupleN}. Null values are represented by an empty string.
+	 */
+	@Override
+	public final String toString ()
+	{
+		return "(" + toPlainString() + ")";
+	}
+
+	@Override
+	public int hashCode ()
+	{
+		return 0;
+	}
+
+	@Override
+	public boolean equals (Object obj)
+	{
+		return obj == this || (obj !=null && obj.getClass().equals(getClass()));
+	}
+
+	/**
+	 * Convenience method implementing objects.equals(object, object), which is not available due to a
+	 * java version restriction.
+	 */
+	protected static boolean equal(Object obj1, Object obj2)
+	{
+		if (obj1 == null) {
+			return obj2 == null;
+		}
+		return obj1.equals(obj2);
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/exception/CatastrophicError.java

@@ -0,0 +1,117 @@
+package com.semmle.util.exception;
+
+import java.util.Arrays;
+
+/**
+ * This is a standard Semmle unchecked exception.
+ * Usage of this should follow the guidelines described in docs/semmle-unchecked-exceptions.md
+ */
+public class CatastrophicError extends NestedError {
+
+	private static final long serialVersionUID = 4132771414092814913L;
+
+	public CatastrophicError(String message) {
+		super(message);
+	}
+
+	public CatastrophicError(Throwable throwable) {
+		super(throwable);
+	}
+
+	public CatastrophicError(String message, Throwable throwable) {
+		super(message,throwable);
+	}
+
+	/**
+	 * Utility method for throwing a {@link CatastrophicError} with the given {@code message} if the given
+	 * {@code condition} is true.
+	 */
+	public static void throwIf(boolean condition, String message)
+	{
+		if (condition) {
+			throw new CatastrophicError(message);
+		}
+	}
+
+	/**
+	 * Utility method for throwing a {@link CatastrophicError} if the given {@code object} is null.
+	 * <p>
+	 * See {@link #throwIfAnyNull(Object...)} which may be more convenient for checking multiple
+	 * arguments.
+	 * </p>
+	 */
+	public static void throwIfNull(Object object)
+	{
+		if (object == null) {
+			throw new CatastrophicError("null object");
+		}
+	}
+
+	/**
+	 * Utility method for throwing a {@link CatastrophicError} with the given {@code message} if the given
+	 * {@code object} is null.
+	 * <p>
+	 * See {@link #throwIfAnyNull(Object...)} which may be more convenient for checking multiple
+	 * arguments.
+	 * </p>
+	 */
+	public static void throwIfNull (Object object, String message)
+	{
+		if (object == null) {
+			throw new CatastrophicError(message);
+		}
+	}
+
+	/**
+	 * Throw a {@link CatastrophicError} if any of the given {@code objects} is null.
+	 * <p>
+	 * If a {@link CatastrophicError} is thrown, its message will indicate <i>all</i> null arguments by index.
+	 * </p>
+	 * <p>
+	 * See {@link #throwIfNull(Object, String)} which may be a fraction more efficient if there's only
+	 * one argument, and allows an 'optional' message parameter.
+	 * </p>
+	 */
+	public static void throwIfAnyNull (Object ... objects)
+	{
+		/*
+		 * Check each argument for nullity, and start building a set of index strings iff at least one
+		 * is non-null
+		 */
+		String[] nullArgs = null;
+		for (int argNum = 0; argNum < objects.length; ++argNum) {
+			if (objects[argNum] == null) {
+				nullArgs = nullArgs == null ? new String[1] : Arrays.copyOf(nullArgs, nullArgs.length+1);
+				nullArgs[nullArgs.length-1] = "" + argNum;
+			}
+		}
+		if (nullArgs != null) {
+			// Compose a message describing which arguments are null
+			StringBuffer strBuf = new StringBuffer();
+			if (nullArgs.length == 0) {
+				strBuf.append("null argument(s)");
+			} else {
+				strBuf.append("null argument" + (nullArgs.length > 1 ? "s: " : ": ") + nullArgs[0]);
+				for (int i = 1; i < nullArgs.length; ++i) {
+					strBuf.append(", " + nullArgs[i]);
+				}
+			}
+			String message = strBuf.toString();
+			throw new CatastrophicError(message);
+		}
+	}
+
+	/**
+	 * Convenience method for use in constructors that assign a parameter to a
+	 * field, assuming the former to be non-null.
+	 *
+	 * @param t A non-null value of type {@code T}.
+	 * @return {@code t}
+	 * @throws CatastrophicError if {@code t} is null.
+	 * @see #throwIfNull(Object)
+	 */
+	public static <T> T nonNull(T t) {
+		throwIfNull(t);
+		return t;
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/exception/Exceptions.java

@@ -0,0 +1,120 @@
+package com.semmle.util.exception;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
+/**
+ * Simple functions for printing exceptions. This is intended for use
+ * in debug output, not for formatting for user consumption
+ */
+public class Exceptions {
+
+	/**
+	 * Compose a String with the same format as that output by {@link Throwable#printStackTrace()}.
+	 */
+	public static String printStackTrace(Throwable t)
+	{
+		StringWriter stringWriter = new StringWriter();
+		t.printStackTrace(new PrintWriter(stringWriter));
+		return stringWriter.toString();
+	}
+
+	/**
+	 * Print an exception in a readable format with all information,
+	 * including the type, message, stack trace, and nested exceptions
+	 */
+	public static String print(Throwable t) {
+		return printDetailed(t, true);
+	}
+
+	/**
+	 * Print an exception in a somewhat readable format fitting on one line.
+	 * Most of the time simply using <code>print</code> is preferable
+	 */
+	public static String printShort(Throwable t) {
+		return printDetailed(t, false);
+	}
+
+	/**
+	 * Ignore an exception. This method does nothing, but should be called
+	 * (with a reasonable message) to document the reason why the exception does
+	 * not need to be used.
+	 */
+	public static void ignore(Throwable e, String message) {
+
+	}
+
+	/**
+	 * Print an exception in a long format, possibly producing multiple
+	 * lines if the appropriate flag is passed
+	 * @param multiline if <code>true</code>, produce multiple lines of output
+	 */
+	private static String printDetailed(Throwable t, boolean multiline) {
+		StringBuilder sb = new StringBuilder();
+
+		Throwable current = t;
+		while (current != null) {
+			printOneException(current, multiline, sb);
+			Throwable cause = current.getCause();
+			if (cause == current)
+				current = null;
+			else
+				current = cause;
+
+			if (current != null) {
+				if (multiline)
+					sb.append("\n\n ... caused by:\n\n");
+				else
+					sb.append(", caused by: ");
+			}
+		}
+
+		return sb.toString();
+	}
+
+	private static void printOneException(Throwable t, boolean multiline, StringBuilder sb) {
+		sb.append(multiline ? t.toString() : t.toString().replace('\n', ' ').replace('\r', ' '));
+		boolean first = true;
+		for (StackTraceElement e : t.getStackTrace()) {
+			if (first)
+				sb.append(multiline ? "\n" : " - [");
+			else
+				sb.append(multiline ? "\n" : ", ");
+			first = false;
+			sb.append(e.toString());
+		}
+		if (!multiline)
+			 sb.append("]");
+	}
+
+	/** A stand-in replacement for `assert` that throws a {@link CatastrophicError} and isn't compiled out. */
+	public static void assertion(boolean cond, String message) {
+		if(!cond)
+			throw new CatastrophicError(message);
+	}
+	
+	/**
+	 * Turn the given {@link Throwable} into a {@link RuntimeException} by wrapping it if necessary.
+	 */
+	public static RuntimeException asUnchecked(Throwable t) {
+		if (t instanceof RuntimeException)
+			return (RuntimeException)t;
+		else
+			return new RuntimeException(t);
+	}
+	
+	/**
+	 * Throws an arbitrary {@link Throwable}, wrapping in a runtime exception if necessary.
+	 * Unlike {@link #asUnchecked} it preserves subclasses of {@link Error}.
+	 */
+	public static <T> T rethrowUnchecked(Throwable t) {
+		if (t instanceof RuntimeException) {
+			throw (RuntimeException) t;
+		} else if (t instanceof Error) {
+			throw (Error) t;
+		}
+		throw new RuntimeException(t);
+	}
+
+
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/exception/InterruptedError.java

@@ -0,0 +1,26 @@
+package com.semmle.util.exception;
+
+/**
+ * An exception thrown in cases where it is impossible to
+ * throw the (checked) Java {@link InterruptedException},
+ * eg. in visitors
+ */
+public class InterruptedError extends RuntimeException {
+
+	private static final long serialVersionUID = 9163340147606765395L;
+
+	public InterruptedError() { }
+
+	public InterruptedError(String message, Throwable cause) {
+		super(message, cause);
+	}
+
+	public InterruptedError(String message) {
+		super(message);
+	}
+
+	public InterruptedError(Throwable cause) {
+		super(cause);
+	}
+
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/exception/NestedError.java

@@ -0,0 +1,47 @@
+package com.semmle.util.exception;
+
+public abstract class NestedError extends RuntimeException {
+
+	private static final long serialVersionUID = -3145876396931008989L;
+
+	public NestedError(String message) {
+		super(message);
+	}
+
+	public NestedError(Throwable throwable) {
+		super(throwable);
+	}
+
+	public NestedError(String message, Throwable throwable) {
+		super(buildMessage(message, throwable), throwable);
+	}
+
+	/**
+	 * Subclasses should not need to call this directly -- just call the
+	 * two-argument super constructor.
+	 */
+	private static String buildMessage(String message, Throwable throwable) {
+		if (throwable == null)
+			return message;
+
+		while (throwable.getCause() != null && throwable.getCause() != throwable)
+			throwable = throwable.getCause();
+		String banner = "eventual cause: " + throwable.getClass().getSimpleName();
+		String rootmsg = throwable.getMessage();
+		if (rootmsg == null) {
+			// Don't amend the banner
+		} else {
+			int p = rootmsg.indexOf('\n');
+			if (p >= 0)
+				rootmsg = rootmsg.substring(0, p) + "...";
+			if (rootmsg.length() > 100)
+				rootmsg = rootmsg.substring(0, 80) + "...";
+			banner += " \"" + rootmsg + "\"";
+		}
+		if (message.contains(banner))
+			return message;
+		else
+			return message + "\n(" + banner + ")";
+	}
+
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/exception/ResourceError.java

@@ -0,0 +1,30 @@
+package com.semmle.util.exception;
+
+/**
+ * This is a standard Semmle unchecked exception.
+ * Usage of this should follow the guidelines described in docs/semmle-unchecked-exceptions.md
+ */
+public class ResourceError extends NestedError {
+
+	private static final long serialVersionUID = 4132771414092814913L;
+
+	public ResourceError(String message) {
+		super(message);
+	}
+
+	@Deprecated // A ResourceError may be presented to the user, so should always have a message
+	public ResourceError(Throwable throwable) {
+		super(throwable);
+	}
+	
+	public ResourceError(String message, Throwable throwable) {
+		super(message,throwable);
+	}
+
+	@Override
+	public String toString() {
+		// The message here should always be meaningful enough that we can return that.
+		return getMessage() != null ? getMessage() : super.toString();
+	}
+	
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/exception/UserError.java

@@ -0,0 +1,46 @@
+package com.semmle.util.exception;
+
+/**
+ * This is a standard Semmle unchecked exception.
+ * Usage of this should follow the guidelines described in docs/semmle-unchecked-exceptions.md
+ */
+public class UserError extends NestedError {
+
+	private static final long serialVersionUID = 4132771414092814913L;
+
+	private final boolean reportAsInfoMessage;
+	
+	public UserError(String message) {
+		this(message, false);
+	}
+	
+	/**
+	 * A user-visible error
+	 * 
+	 * @param message The message to display
+	 * @param reportAsInfoMessage If <code>true</code>, report as information only - not an error
+	 */
+	public UserError(String message, boolean reportAsInfoMessage) {
+		super(message);
+		this.reportAsInfoMessage = reportAsInfoMessage;
+	}
+	
+	public UserError(String message, Throwable throwable) {
+		super(message,throwable);
+		this.reportAsInfoMessage = false;
+	}
+	
+	/**
+	 * If <code>true</code>, report the message without interpreting it as a fatal error
+	 */
+	public boolean reportAsInfoMessage() {
+		return reportAsInfoMessage;
+	}
+
+	@Override
+	public String toString() {
+		// The message here should always be meaningful enough that we can return that.
+		return getMessage() != null ? getMessage() : super.toString();
+	}
+
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/expansion/ExpansionEnvironment.java

@@ -0,0 +1,893 @@
+package com.semmle.util.expansion;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.semmle.util.data.StringUtil;
+import com.semmle.util.exception.CatastrophicError;
+import com.semmle.util.exception.ResourceError;
+import com.semmle.util.exception.UserError;
+import com.semmle.util.files.FileUtil;
+import com.semmle.util.process.Builder;
+import com.semmle.util.process.Env;
+import com.semmle.util.process.Env.Var;
+import com.semmle.util.process.LeakPrevention;
+
+/**
+ * An environment for performing variable expansions.
+ *
+ * <p>
+ * The environment is defined by a set of variable definitions, which are
+ * name/value pairs of strings. Once this has been populated (via the
+ * {@link #defineVar(String, String)} and {@link #defineVars(Map)} methods),
+ * arbitrary strings can be expanded.
+ * </p>
+ *
+ * <p>
+ * Two modes of expansion are supported:
+ * </p>
+ * <ul>
+ * <li>String mode ({@link #strExpand(String)}): The result is intended to be a
+ * single string.</li>
+ * <li>List mode ({@link #listExpand(String)}): The result will be interpreted
+ * as a command line, and hence is a list of strings.
+ * </ul>
+ *
+ * <p>
+ * Variables are referenced by <code>${name}</code> to trigger a string-mode
+ * expansion, and by <code>${=name}</code> to trigger a list-mode expansion.
+ * This makes {@code $} a meta-character, and so it has to be escaped; the
+ * escape sequence for it is <code>${}</code>.
+ * </p>
+ *
+ * <p>
+ * In list mode, strings are split in a platform-independent way similar (but
+ * not identical) to normal shell argument splitting. Runs of white-space
+ * separate arguments, and double-quotes can be used to protect whitespace from
+ * splitting. The escape character is backslash. All of these metacharacters
+ * have no special meaning in string mode.
+ * </p>
+ *
+ * <p>
+ * The {@code define*} and {@link #doNotExpand(String...)} methods of this
+ * class are not thread-safe; they mutate instance state in an unsynchronized
+ * way. By contrast, the expansion methods ({@link #strExpand(String)},
+ * {@link #strExpandVar(String)}, {@link #listExpand(String)},
+ * {@link #listExpandVar(String)} and {@link #varLookup(String)})
+ * are thread safe relative to each
+ * other. This means that it's fine to construct an expansion environment once,
+ * and then use it from multiple threads concurrently, as long as no new variables
+ * are defined. In addition, {@link #validate(String)} is safe to call once an
+ * {@link ExpansionEnvironment} is fully initialised, even concurrently.
+ * </p>
+ *
+ * <p>
+ * Upon encountering any error (malformed variable expansion, malformed quoted
+ * string (in list mode), reference to unknown variable, cyclic variable
+ * definitions), the {@link #strExpand(String)} and {@link #listExpand(String)}
+ * methods will throw {@link UserError} with a suitable message.
+ * </p>
+ *
+ * <p>
+ * As an advanced feature, command substitutions can be supported. They take the
+ * form of <code>$(cmd arg1 arg2)</code> for string-mode expansion, and
+ * <code>$(=cmd arg1
+ * arg2)</code> for list-mode. The contents of the <code>$(..)</code> operator
+ * undergo normal splitting, and are then run as a new process with the given
+ * list of arguments. The working directory is unspecified, and it is an error
+ * to depend upon it. A non-zero exit code, or a non-empty {@code stderr} stream
+ * of the command, will result in a {@link UserError} indicating that something
+ * went wrong; otherwise, the {@code stdout} output is collected and substituted
+ * (possibly undergoing splitting, in the second form).
+ * </p>
+ */
+public class ExpansionEnvironment {
+
+	/**
+	 * A source for variable definitions to be used in an expansion environment.
+	 */
+	public static interface VariableSource {
+		/**
+		 * A callback which is expected to add all variables in the source to
+		 * the given environment.
+		 *
+		 * @param env
+		 *            The environment that should be filled in.
+		 */
+		public void fillIn(ExpansionEnvironment env);
+	}
+
+	private final Map<String, String> vars = new LinkedHashMap<String, String>();
+
+	private final Set<String> unexpandedVars = new LinkedHashSet<String>();
+
+	private final boolean commandSubstitutions;
+
+	/**
+	 * Construct an empty {@link ExpansionEnvironment}.
+	 */
+	public ExpansionEnvironment(boolean commandSubstitutions) {
+		this.commandSubstitutions = commandSubstitutions;
+	}
+
+	/**
+	 * This the old default constructor, which always enables command substitutions.
+	 * <b>Doing so is a security risk</b> whenever the string you expand may come
+	 * from an untrusted source, so you should only do that when you explicitly want
+	 * to do it and have decided that it is safe. (And then use the constructor that
+	 * has an explicit argument to say so!)
+	 */
+	@Deprecated
+	public ExpansionEnvironment() {
+		this(true);
+	}
+
+	/**
+	 * Construct an environment based on an existing map.
+	 */
+	public ExpansionEnvironment(boolean commandSubstitutions, Map<String, String> vars) {
+		this(commandSubstitutions);
+		this.vars.putAll(vars);
+	}
+
+	/**
+	 * Construct a copy of an existing {@link ExpansionEnvironment}.
+	 */
+	public ExpansionEnvironment(ExpansionEnvironment other) {
+		this(other.commandSubstitutions);
+		this.vars.putAll(other.vars);
+		this.unexpandedVars.addAll(other.unexpandedVars);
+	}
+
+	/**
+	 * Add a set of variable definitions to this environment.
+	 *
+	 * @param vars
+	 *            A mapping from variable names to variable values. Recursive
+	 *            variable references are allowed, but cycles are an error.
+	 */
+	public void defineVars(Map<String, String> vars) {
+		this.vars.putAll(vars);
+	}
+
+	/**
+	 * Add the specified variable definition to this environment.
+	 *
+	 * @param name
+	 *            A variable name.
+	 * @param value
+	 *            The value that the variable should expand to. References to
+	 *            other variables or expansions are allowed, but cycles are an
+	 *            error.
+	 */
+	public void defineVar(String name, String value) {
+		this.vars.put(name, value);
+	}
+
+	/**
+	 * Try to load a file as a Java properties file and add all of its key/value
+	 * pairs as variable definitions.
+	 *
+	 * @param vars
+	 *            A {@link File} that will be loaded as a Java properties file,
+	 *            if it exists. May be <code>null</code> or a file whose
+	 *            existence has not been checked.
+	 * @throws ResourceError
+	 *             if the file exists but can't be read, or exists as a
+	 *             directory, or reading it fails.
+	 */
+	public void defineVarsFromFile(File vars) {
+		if (vars == null || !vars.exists())
+			return;
+
+		if (vars.isDirectory())
+			throw new ResourceError(vars
+					+ " is a directory, cannot load variables from it.");
+
+		Properties properties = FileUtil.loadProperties(vars);
+		for (String key : properties.stringPropertyNames())
+			defineVar(key, properties.getProperty(key));
+	}
+
+	/**
+	 * Add a variable definition of {@code env.foo=bar} for each system
+	 * environment variable {@code foo=bar}. Typically it is desirable to allow
+	 * the environment to override previously specified variables, so this
+	 * should be called once all other variables have been defined.
+	 *
+	 * <p>
+	 * The values of variables taken from the environment are escaped to prevent
+	 * recursive expansion; in particular, this prevents accidental command
+	 * execution if a command substitution is encountered in the environment.
+	 * </p>
+	 */
+	public void defineVarsFromEnvironment(Env environment) {
+		String extraVars = environment.get(Var.ODASA_EXTRA_VARIABLES);
+		if (extraVars != null)
+			defineVarsFromFile(new File(extraVars));
+
+		for (Entry<String, String> var : environment.getenv().entrySet())
+			defineVar("env." + var.getKey(), var.getValue().replace("$", "${}"));
+
+		environment.addEnvironmentToNewEnv(this);
+	}
+
+	/**
+	 * Indicate that references to the given set of variable names should not be
+	 * expanded. This means that they need not be defined, and the output will
+	 * contain the literal variable expansion sequences.
+	 *
+	 * @param vars
+	 *            A list of variable names.
+	 */
+	public void doNotExpand(String... vars) {
+		for (String var : vars)
+			unexpandedVars.add(var);
+	}
+
+	/**
+	 * Supply a "default value" for a variable, meaning that the variable will
+	 * be set to the given default value if it hasn't already been defined. No
+	 * change is made to this environment if a definition exists.
+	 * @param var A variable name.
+	 * @param defaultValue The default value for the named variable.
+	 */
+	public void setDefault(String var, String defaultValue) {
+		if (!vars.containsKey(var))
+			vars.put(var, defaultValue);
+	}
+
+	/**
+	 * Expand the given string in "string mode", resolving variable references
+	 * and command substitutions.
+	 */
+	public String strExpand(String s) {
+		try {
+			return new Expander().new ExpansionParser(s).parseAsString().expandAsString();
+		} catch (UserError e) {
+			throw new UserError("Failed to expand '" + s + "'.", e);
+		}
+	}
+
+	/**
+	 * Expand the given string in "list mode", resolving variable references and
+	 * command substitutions.
+	 */
+	public List<String> listExpand(String s) {
+		try {
+			return new Expander().new ExpansionParser(s).parseAsList().expandAsList();
+		} catch (UserError e) {
+			throw new UserError("Failed to expand '" + s
+					+ "' as an argument list.", e);
+		}
+	}
+
+	/**
+	 * Expand the given variable fully in "string mode", resolving variable
+	 * references and command substitutions. The entire string is interpreted as
+	 * the name of the initial variable.
+	 */
+	public String strExpandVar(String varName) {
+		return new Expander().new Variable(varName).expandAsString();
+	}
+
+	/**
+	 * Expand the given variable fully in "list mode", resolving variable
+	 * references and command substitutions. The entire string is interpreted as
+	 * the name of the initial variable.
+	 */
+	public List<String> listExpandVar(String varName) {
+		return new Expander().new SplitVariable(varName).expandAsList();
+	}
+
+	/**
+	 * Validate the given string for expansion. This verifies the absence of
+	 * parse errors, and the fact that all directly referenced variables are
+	 * defined by this environment.
+	 *
+	 * <p>
+	 * Expansion using {@link #strExpand(String)} or {@link #listExpand(String)}
+	 * may still not succeed, if there are semantic errors (like circular
+	 * variable definitions) or a command substitution introduces a reference to
+	 * an undefined variable.
+	 * </p>
+	 *
+	 * @param str
+	 *            A string that should be validated.
+	 * @throws UserError
+	 *             if validation fails, with a suitable error message.
+	 */
+	public void validate(String str) {
+		new Expander().new ExpansionParser(str).parseAsList().validate();
+	}
+
+	/**
+	 * Look up the (raw) value of a given variable, without performing expansion
+	 * on it.
+	 *
+	 * @param name
+	 *            The variable name.
+	 * @return The value that this variable is mapped to.
+	 * @throws UserError
+	 *             if the variable is not defined.
+	 */
+	public synchronized String varLookup(String name) {
+		String value = vars.get(name);
+		if (value == null) {
+			ArrayList<String> available = new ArrayList<String>(vars.keySet());
+			Collections.sort(available);
+			throw new UserError("Attempting to expand unknown variable: "
+					+ name + ", available variables are: " + available);
+		}
+		return value;
+	}
+
+	/**
+	 * Check whether this environment defines a variable of the given name, without
+	 * performing expansion on it -- such full expansion may still fail.
+	 *
+	 * @param name The variable name.
+	 * @return <code>true</code> if this environment contains a direct definition
+	 */
+	public boolean definesVar(String name) {
+		return vars.containsKey(name);
+	}
+
+	private static class ExpansionTokeniser {
+		/**
+		 * The delimiters which should be returned as their own tokens. Order of
+		 * alternatives matters! The recognised tokens are, in order:
+		 *
+		 * <ul>
+		 * <li>{@code \\}</li>
+		 * <li>{@code \"}</li>
+		 * <li>{@code "}</li>
+		 * <li><code>${}</code></li>
+		 * <li><code>${=</code></li>
+		 * <li><code>${</code></li>
+		 * <li><code>$(=</code></li>
+		 * <li><code>$(</code></li>
+		 * <li><code>$</code></li>
+		 * <li><code>}</code></li>
+		 * <li><code>)</code></li>
+		 * <li>Runs of whitespace.</li>
+		 * </ul>
+		 *
+		 * <p>
+		 * By defining the alternatives in this order, longer matches will be
+		 * preferred, so that checking for escape sequences is easy. Note that
+		 * in the regular expression source, a literal {@code \} must undergo
+		 * two levels of escaping: Java strings and regular expression
+		 * metacharacters; it thus becomes {@code \\\\}.
+		 */
+		private static final Pattern delims = Pattern
+				.compile("\\\\\\\\|\\\\\"|\"|\\$\\{\\}|\\$\\{=|\\$\\{|"
+						+ "\\$\\(=|\\$\\(|\\$|\\}|\\)|\\s+");
+
+		private final List<String> tokens = new ArrayList<String>();
+		private final int[] positions;
+		private int nextToken = 0;
+
+		public ExpansionTokeniser(String str) {
+			Matcher matcher = delims.matcher(str);
+			StringBuffer tmp = new StringBuffer();
+			while (matcher.find()) {
+				matcher.appendReplacement(tmp, "");
+				if (tmp.length() > 0) {
+					tokens.add(tmp.toString());
+					tmp = new StringBuffer();
+				}
+				tokens.add(matcher.group());
+			}
+			matcher.appendTail(tmp);
+			if (tmp.length() > 0)
+				tokens.add(tmp.toString());
+
+			positions = new int[tokens.size()];
+			int pos = 0;
+			for (int i = 0; i < tokens.size(); i++) {
+				positions[i] = pos;
+				pos += tokens.get(i).length();
+			}
+		}
+
+		public boolean hasMoreTokens() {
+			return nextToken < tokens.size();
+		}
+
+		public String nextToken() {
+			return tokens.get(nextToken++);
+		}
+
+		public boolean isDelimiter(String token) {
+			return delims.matcher(token).matches();
+		}
+
+		public int pos() {
+			return positions[nextToken - 1] + 1;
+		}
+	}
+
+	/**
+	 * A wrapper around the various expansion classes, holding some expansion
+	 * state to detect things like circular variable definitions.
+	 */
+	private class Expander {
+
+		private final Set<String> expansionsInProgress = new LinkedHashSet<String>();
+
+		/**
+		 * A string expansion. This can be a literal string, a variable reference or
+		 * a command substitution; the latter two can optionally be "split". Each
+		 * expansion can be interpreted to yield a single string or a list of
+		 * strings (typically as program arguments).
+		 */
+		abstract class Expansion {
+			public abstract String expandAsString();
+
+			public abstract List<String> expandAsList();
+
+			public abstract void validate();
+		}
+
+		class Sentence extends Expansion {
+			private final List<List<Expansion>> words = new ArrayList<List<Expansion>>();
+
+			public Sentence(List<List<Expansion>> words) {
+				this.words.addAll(words);
+			}
+
+			@Override
+			public void validate() {
+				for (List<Expansion> expansions : words)
+					for (Expansion expansion : expansions)
+						expansion.validate();
+			}
+
+			private String expandWord(List<Expansion> word) {
+				StringBuilder result = new StringBuilder();
+				for (Expansion e : word)
+					result.append(e.expandAsString());
+				return result.toString();
+			}
+
+			@Override
+			public String expandAsString() {
+				StringBuilder result = new StringBuilder();
+
+				for (List<Expansion> word : words) {
+					if (result.length() > 0)
+						result.append(' ');
+					result.append(expandWord(word));
+				}
+
+				return result.toString();
+			}
+
+			@Override
+			public List<String> expandAsList() {
+				List<String> result = new ArrayList<String>();
+
+				for (List<Expansion> word : words) {
+					List<List<String>> segments = new ArrayList<List<String>>();
+					for (Expansion e : word) {
+						segments.add(e.expandAsList());
+					}
+					result.addAll(glue(segments));
+				}
+
+				return result;
+			}
+
+			/**
+			 * This is a non-quadratic implementation of the following Haskell code:
+			 *
+			 * <pre>
+			 * <code>
+			 * glue :: [[String]] -&gt; [String]
+			 * glue = foldr join []
+			 *     where join [] xs = xs
+			 *           join xs [] = xs
+			 *           join xs ys = init xs ++ [last xs ++ head ys] ++ tail ys
+			 * </code>
+			 * </pre>
+			 */
+			private List<String> glue(List<List<String>> segments) {
+				String trailingWord = null;
+				List<String> result = new ArrayList<String>();
+				for (List<String> segment : segments)
+					trailingWord = glue_join_accum(result, segment, trailingWord);
+
+				if (trailingWord != null)
+					result.add(trailingWord);
+
+				return result;
+			}
+
+			private String glue_join_accum(List<String> result,
+					List<String> segment, String trailingWord) {
+				int n = segment.size();
+				switch (n) {
+				case 0:
+					return trailingWord;
+				case 1:
+					return combine(trailingWord, segment.get(0));
+				default:
+					result.add(combine(trailingWord, segment.get(0)));
+					result.addAll(segment.subList(1, n - 1));
+					return segment.get(n - 1);
+				}
+			}
+
+			private String combine(String a, String b) {
+				if (a == null)
+					return b;
+				return a + b;
+			}
+		}
+
+		class Literal extends Expansion {
+			private final String value;
+
+			public Literal(String value) {
+				this.value = value;
+			}
+
+			@Override
+			public void validate() {
+				// Always valid.
+			}
+
+			@Override
+			public String expandAsString() {
+				return value;
+			}
+
+			@Override
+			public List<String> expandAsList() {
+				return Collections.singletonList(value);
+			}
+		}
+
+		class QuotedString extends Sentence {
+			public QuotedString(List<Expansion> content) {
+				super(Collections.singletonList(content));
+			}
+
+			@Override
+			public List<String> expandAsList() {
+				return Collections.singletonList(this.expandAsString());
+			}
+		}
+
+		class Variable extends Expansion {
+			protected final String name;
+
+			public Variable(String name) {
+				this.name = name;
+			}
+
+			@Override
+			public void validate() {
+				varLookup(name); // Will throw if variable is undefined.
+			}
+
+			protected void startExpanding(String name) {
+				if (!expansionsInProgress.add(name))
+					throw new UserError("Circular expansion of variable " + name);
+			}
+
+			protected void doneWith(String name) {
+				if (!expansionsInProgress.remove(name))
+					throw new CatastrophicError("Not currently expanding " + name);
+			}
+
+			protected String ref() {
+				return "${" + name + "}";
+			}
+
+			@Override
+			public final String expandAsString() {
+				if (unexpandedVars.contains(name))
+					return ref();
+				startExpanding(name);
+				String result = expandAsStringImpl();
+				doneWith(name);
+				return result;
+			}
+
+			public String expandAsStringImpl() {
+				// Not calling ExpansionEnvironment.strExpand(), since
+				// we must run in the same enclosing instance of Expander.
+				return new ExpansionParser(varLookup(name)).parseAsString().expandAsString();
+			}
+
+			@Override
+			public final List<String> expandAsList() {
+				if (unexpandedVars.contains(name))
+					return Collections.singletonList(ref());
+				startExpanding(name);
+				List<String> result = expandAsListImpl();
+				doneWith(name);
+				return result;
+			}
+
+			public List<String> expandAsListImpl() {
+				return Collections.singletonList(expandAsStringImpl());
+			}
+		}
+
+		class SplitVariable extends Variable {
+			public SplitVariable(String name) {
+				super(name);
+			}
+
+			@Override
+			protected String ref() {
+				return "${=" + name + "}";
+			}
+
+			@Override
+			public String expandAsStringImpl() {
+				return StringUtil.glue(" ", expandAsListImpl());
+			}
+
+			@Override
+			public List<String> expandAsListImpl() {
+				return listExpand(varLookup(name));
+			}
+		}
+
+		class Command extends Expansion {
+			private final Sentence argv;
+
+			public Command(List<List<Expansion>> args) {
+				this.argv = new Sentence(args);
+			}
+
+			@Override
+			public void validate() {
+				argv.validate();
+			}
+
+			protected String run() {
+				List<String> args = argv.expandAsList();
+				ByteArrayOutputStream result = new ByteArrayOutputStream();
+				ByteArrayOutputStream err = new ByteArrayOutputStream();
+				Builder builder = new Builder(args, result, err);
+				builder.setLeakPrevention(LeakPrevention.ALL);
+				try {
+					int exitCode = builder.execute();
+					if (exitCode != 0)
+						throw new UserError("Exit code " + exitCode
+								+ " from command "
+								+ builder.toString());
+					if (err.size() > 0)
+						throw new UserError("Command \""
+								+ builder.toString()
+								+ "\" produced output on stderr: " + err.toString());
+				} catch (RuntimeException e) {
+					throw new UserError("Could not execute command "
+							+ builder.toString(), e);
+				}
+				return result.toString();
+			}
+
+			@Override
+			public String expandAsString() {
+				return run();
+			}
+
+			@Override
+			public List<String> expandAsList() {
+				return Collections.singletonList(expandAsString());
+			}
+		}
+
+		class SplitCommand extends Command {
+			public SplitCommand(List<List<Expansion>> argv) {
+				super(argv);
+			}
+
+			@Override
+			public String expandAsString() {
+				return StringUtil.glue(" ", expandAsList());
+			}
+
+			@Override
+			public List<String> expandAsList() {
+				return new ExpansionParser(run()).splitAsString().expandAsList();
+			}
+		}
+
+		private class ExpansionParser {
+			private final ExpansionTokeniser tokens;
+
+			public ExpansionParser(String str) {
+				tokens = new ExpansionTokeniser(str);
+			}
+
+			public Sentence parseAsString() {
+				List<List<Expansion>> words = new ArrayList<List<Expansion>>();
+				words.add(parseTerminatedString(null));
+				return new Sentence(words);
+			}
+
+			public Sentence parseAsList() {
+				return new Sentence(parseTerminatedList(null, false));
+			}
+
+			public Sentence splitAsString() {
+				return new Sentence(parseTerminatedList(null, true));
+			}
+
+			private List<Expansion> parseTerminatedString(String terminator) {
+				List<Expansion> result = new ArrayList<Expansion>();
+
+				while (tokens.hasMoreTokens()) {
+					String next = tokens.nextToken();
+					if (next.equals(terminator)) {
+						return result;
+					} else if (next.equals("\\\"")) {
+						result.add(new Literal("\""));
+					} else if (next.equals("\\\\")) {
+						result.add(new Literal("\\"));
+					} else if (!tryParseExpansion(result, next)) {
+						result.add(new Literal(next));
+					}
+				}
+
+				if (terminator != null)
+					throw new UserError(
+							"Premature end of input while looking for matching '"
+									+ terminator + "'.");
+
+				return result;
+			}
+
+			private List<List<Expansion>> parseTerminatedList(String terminator,
+					boolean noExpansions) {
+				List<List<Expansion>> result = new ArrayList<List<Expansion>>();
+
+				List<Expansion> accum = new ArrayList<Expansion>();
+				boolean mustSeeSpace = false;
+				while (tokens.hasMoreTokens()) {
+					String next = tokens.nextToken();
+					if (next.equals(terminator)) {
+						if (accum.size() > 0)
+							result.add(accum);
+						return result;
+					} else if (mustSeeSpace
+							&& !Character.isWhitespace(next.charAt(0))) {
+						throw new UserError("The quoted string ending at "
+								+ tokens.pos()
+								+ " must be surrounded by whitespace.");
+					} else if (next.length() > 0
+							&& Character.isWhitespace(next.charAt(0))) {
+						mustSeeSpace = false;
+						if (accum.size() > 0) {
+							result.add(accum);
+							accum = new ArrayList<Expansion>();
+						}
+					} else if (next.equals("\"")) {
+						if (!accum.isEmpty())
+							throw new UserError(
+									"At position "
+											+ tokens.pos()
+											+ ", the quote should "
+											+ "either be preceded by a space (if it is intended to start an argument) "
+											+ "or escaped as \\\".");
+						accum.add(new QuotedString(parseTerminatedString("\"")));
+						result.add(accum);
+						accum = new ArrayList<Expansion>();
+						mustSeeSpace = true;
+					} else if (next.equals("\\\"")) {
+						// An escaped quote means a literal quote.
+						accum.add(new Literal("\""));
+					} else if (next.equals("\\\\")) {
+						// An escaped backslash means a literal backslash.
+						accum.add(new Literal("\\"));
+					} else if (noExpansions || !tryParseExpansion(accum, next)) {
+						accum.add(new Literal(next));
+					}
+				}
+
+				if (terminator != null)
+					throw new UserError(
+							"Premature end of expansion while looking for '"
+									+ terminator + "'.");
+
+				if (accum.size() > 0)
+					result.add(accum);
+
+				return result;
+			}
+
+			private boolean tryParseExpansion(List<Expansion> result,
+					String curToken) {
+				if (curToken.equals("${}")) {
+					result.add(new Literal("$"));
+				} else if (curToken.equals("$(=") && commandSubstitutions) {
+					result.add(new SplitCommand(parseTerminatedList(")", false)));
+				} else if (curToken.equals("$(") && commandSubstitutions) {
+					result.add(new Command(parseTerminatedList(")", false)));
+				} else if (curToken.equals("${=")) {
+					result.add(new SplitVariable(parseVarName()));
+				} else if (curToken.equals("${")) {
+					result.add(new Variable(parseVarName()));
+				} else if (curToken.equals("$")) {
+					throw new UserError(
+							"Malformed expansion: A standalone '$' character should be escaped as '${}'.");
+				} else {
+					return false;
+				}
+				return true;
+			}
+
+			protected String parseVarName() {
+				if (!tokens.hasMoreTokens())
+					throw new UserError(
+							"Malformed variable substitution: stray '${' at " + tokens.pos());
+				String name = tokens.nextToken();
+				if (tokens.isDelimiter(name))
+					throw new UserError(
+							"Malformed variable substitution: Unexpected '" + name
+							+ "' at " + tokens.pos());
+				if (!tokens.hasMoreTokens())
+					throw new UserError(
+							"Malformed variable substitution for '" + name +
+							"': Missing '}' at " + tokens.pos());
+				String next = tokens.nextToken();
+				if (!next.equals("}"))
+					throw new UserError(
+							"Malformed variable substitution: Expecting '}' at "
+									+ tokens.pos() + ", found '" + next + "'.");
+				return name;
+			}
+		}
+	}
+
+	/**
+	 * Resolve a path. Any variables in the path will be expanded. If
+	 * the path is an absolute path after expansion, it is returned as is.
+	 * Otherwise, it is combined with the given base path.
+	 */
+	public File expandPath(File base, String path) {
+		String expanded = strExpand(path);
+		if (FileUtil.isAbsolute(expanded)) {
+			return new File(expanded);
+		} else {
+			return FileUtil.fileRelativeTo(base, expanded);
+		}
+	}
+
+	/**
+	 * Escape a string so that any '$'s inside it will be interpreted literally, rather than
+	 * as parts of variable references.
+	 */
+	public static String escape(String base) {
+		return base.replace("$", "${}");
+	}
+
+	/**
+	 * Escape {@code argument} as an argument, so that any {@code $}, {@code \} or {@code "} is interpreted literally.
+	 *
+	 * @param argument - the String to escape.
+	 * @return the escaped String.
+	 */
+	public static String escapeArgument(String argument) {
+		return escape(argument).replaceAll(Matcher.quoteReplacement("\\"), Matcher.quoteReplacement("\\\\")).replaceAll(Matcher.quoteReplacement("\""), Matcher.quoteReplacement("\\\""));
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/extraction/SpecFileEntry.java

@@ -0,0 +1,48 @@
+package com.semmle.util.extraction;
+
+import java.io.File;
+import java.util.List;
+
+import com.semmle.util.data.StringUtil;
+
+public class SpecFileEntry {
+	private final File trapFolder;
+	private final File sourceArchivePath;
+	private final List<String> patterns;
+	
+	public SpecFileEntry(File trapFolder, File sourceArchivePath, List<String> patterns) {
+		this.trapFolder = trapFolder;
+		this.sourceArchivePath = sourceArchivePath;
+		this.patterns = patterns;
+	}
+	
+	public boolean matches(String path) {
+		boolean matches = false;
+		for (String pattern : patterns) {
+			if (pattern.startsWith("-")) {
+				if (path.startsWith(pattern.substring(1)))
+					matches = false;
+			} else {
+				if (path.startsWith(pattern))
+					matches = true;
+			}
+		}
+		return matches;
+	}
+	
+	public File getTrapFolder() {
+		return trapFolder;
+	}
+	
+	public File getSourceArchivePath() {
+		return sourceArchivePath;
+	}
+	
+	@Override
+	public String toString() {
+		return
+			"TRAP_FOLDER=" + trapFolder + "\n" + 
+			"SOURCE_ARCHIVE=" + sourceArchivePath + "\n" + 
+			StringUtil.glue("\n", patterns);
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/files/PathMatcher.java

@@ -0,0 +1,160 @@
+package com.semmle.util.files;
+
+import java.util.regex.Pattern;
+
+import com.semmle.util.data.StringUtil;
+
+/**
+ * Utility class to match a string to a pattern, which can either be
+ * an ant-like include/exclude pattern (with wildcards), or a rsync-like
+ * pattern.
+ * <p>
+ * In ant-like mode:
+ * <ul>
+ *   <li>'**' matches zero or more characters (most notably including '/'). 
+ *   <li>'*' matches zero or more characters except for '/'. 
+ *   <li>'?' matches any character (other than '/').
+ * </ul>
+ * <p>
+ * In rsync-like mode:
+ * <ul>
+ *   <li>A pattern is matched only at the root if it starts with '/', and otherwise
+ *       it is matched against each level of the directory tree.
+ *   <li>'**', '*' and '?' have the same meaning as for ant.
+ *   <li>Other rsync features (like [:..:] groups and backslash-escapes) are not supported.
+ * </ul>
+ */
+public class PathMatcher {
+	
+	public enum Mode {
+		Ant, Rsync;
+	}
+	
+	private final Mode mode;
+	private final Pattern pattern;
+	private final String originalPattern;
+	
+	/**
+	 * Create a {@link PathMatcher}.
+	 * 
+	 * @param pattern An ant-like pattern
+	 */
+	public PathMatcher(String pattern) {
+		this(Mode.Ant, pattern);
+	}
+	
+	/** Create a {@link PathMatcher}.
+	 *
+	 * @param mode The {@link Mode} to use
+	 * @param pattern A pattern, interpreted as ant-like or rsync-like depending on
+	 *     the value of {@code mode}
+	 */
+	public PathMatcher(Mode mode, String pattern) {
+		this.mode = mode;
+		this.originalPattern = pattern;
+		StringBuilder b = new StringBuilder();
+		toRegex(b, pattern);
+		this.pattern = Pattern.compile(b.toString());
+	}
+
+	/** Create a {@link PathMatcher}.
+	 * 
+	 * @param patterns Several ant-like patterns
+	 */
+	public PathMatcher(Iterable<String> patterns) {
+		this(Mode.Ant, patterns);
+	}
+	
+	/** Create a {@link PathMatcher}.
+	 * 
+	 * @param mode The {@link Mode} to use.
+	 * @param patterns Several patterns, interpreted as ant-like or rsync-like depending
+	 *     on the value of {@code mode}.
+	 */
+	public PathMatcher(Mode mode, Iterable<String> patterns) {
+		this.mode = mode;
+		this.originalPattern = patterns.toString();
+		StringBuilder b = new StringBuilder();
+		for (String pattern : patterns) {
+			if (b.length() > 0)
+				b.append('|');
+			toRegex(b, pattern);
+		}
+		this.pattern = Pattern.compile(b.toString());
+	}
+
+	private void toRegex(StringBuilder b, String pattern) {
+		if (pattern.length() == 0) return;
+		//normalize pattern path separators
+		pattern = pattern.replace('\\', '/');
+		//replace double slashes
+		pattern = pattern.replaceAll("//+", "/");
+		// escape
+		pattern = StringUtil.escapeStringLiteralForRegexp(pattern, "*?");
+
+		// for ant, ending with '/' is shorthand for "/**"
+		if (mode == Mode.Ant && pattern.endsWith("/")) pattern = pattern + "**";
+
+		// replace "**/" with (^|.*/)"
+		// replace "**" with ".*"
+		// replace "*" with "[^/]*
+		// replace "?" with "[^/]"
+		int i = 0;
+		
+		// In rsync-mode, a leading slash is an 'anchor' -- the pattern is only matched
+		// when rooted at the start of the path. This is the default behaviour for ant-like
+		// patterns.
+		if (mode == Mode.Rsync) {
+			if (pattern.charAt(0) == '/') {
+				// The slash is just anchoring, and may actually be missing
+				// in the case of a relative path.
+				b.append("/?");
+				i++;
+			} else {
+				// Non-anchored rsync pattern: the pattern can match at any level in the tree.
+				b.append("(.*/)?");
+			}
+		}
+
+		while (i < pattern.length()) {
+			char c = pattern.charAt(i);
+			if (c == '*' && i < pattern.length() - 2 && pattern.charAt(i+1) == '*' && pattern.charAt(i+2) == '/') {
+				b.append("(?:^|.*/)");
+				i += 3;
+			}
+			else if (c == '*' && i < pattern.length() - 1 && pattern.charAt(i+1) == '*') {
+				b.append(".*");
+				i += 2;
+			}
+			else if(c == '*') {
+				b.append("[^/]*");
+				i++;
+			}
+			else if(c == '?') {
+				b.append("[^/]");
+				i++;
+			}
+			else {
+				b.append(c);
+				i++;
+			}
+		}
+	}
+	
+	/**
+	 * Match the specified path against a shell pattern. The path is normalised by replacing '\' with '/'.
+	 * @param path The path to match.
+	 */
+	public boolean matches(String path) {
+		// normalise path
+		path = path.replace('\\', '/');
+		if(path.endsWith("/"))
+			path = path.substring(0, path.length()-1);
+		return pattern.matcher(path).matches();
+	}
+	
+	@Override
+	public String toString() {
+		return "Matches " + originalPattern + " [" + pattern + "]";
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/io/BufferedLineReader.java

@@ -0,0 +1,103 @@
+package com.semmle.util.io;
+
+import java.io.BufferedReader;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Reader;
+
+import com.semmle.util.files.FileUtil;
+
+/**
+ * A custom buffered reader akin to {@link BufferedReader}, except that it preserves
+ * line terminators (and so its {@code readLine()} method is called
+ * {@link #readLineAndTerminator()}). The other {@link Reader} methods should not
+ * be called, and will throw.
+ */
+public class BufferedLineReader implements Closeable {
+	private final char[] buffer = new char[8192];
+	private  int nextChar = 0, nChars = 0;
+	private final Reader in;
+	
+	public BufferedLineReader(Reader in) {
+		this.in = in;
+	}
+
+	/**
+	 * Read the string up to and including the next CRLF or LF terminator. This method
+	 * may return a non-terminated string at EOF, or if a line is too long to fit in the
+	 * internal buffer. Calls will block until enough data has been read to fill the
+	 * buffer or find a line terminator.
+	 * @return The next line (or buffer-full) of text.
+	 * @throws IOException if the underlying stream throws.
+	 */
+	public String readLineAndTerminator() throws IOException {
+		int terminator = findNextLineTerminator();
+		if (terminator == -1)
+			return null;
+		String result = new String(buffer, nextChar, terminator - nextChar + 1);
+		nextChar = terminator + 1;
+		return result;
+	}
+	
+	/**
+	 * Get the index of the last character that should be included in the next line.
+	 * Usually, this is the LF in a LF or CRLF line terminator, but it might be the
+	 * end of the buffer (if it is full, and no newlines are present), or it may be
+	 * -1 (but only if EOF has been reached, and the buffer is currently empty).
+	 * The first character of the line is pointed to by {@link #nextChar}, which
+	 * may be modified by this method if the buffer is refilled.
+	 */
+	private int findNextLineTerminator() throws IOException {
+		int alreadyChecked = 0;
+		do {
+			for (int i = nextChar + alreadyChecked; i < nChars; i++) {
+				if (buffer[i] == '\r' && i+1 < nChars && buffer[i+1] == '\n')
+					return i+1; // CRLF
+				else if (buffer[i] == '\n')
+					return i; // LF
+			}
+			
+			// We didn't find a full newline in the existing buffer: Try to fill.
+			alreadyChecked = nChars - nextChar;
+			int newlyRead = fill();
+			if (newlyRead <= 0)
+				return nChars - 1;
+		} while (true);
+	}
+	
+	/**
+	 * Block until at least one character from the underlying stream is read,
+	 * or EOF is reached.
+	 */
+	private int fill() throws IOException {
+		if (nextChar >= nChars) {
+			// No unread characters.
+			nextChar = 0;
+			nChars = 0;
+		} else if (nextChar > 0) {
+			// Some unread characters.
+			System.arraycopy(buffer, nextChar, buffer, 0, nChars - nextChar);
+			nChars = nChars - nextChar;
+			nextChar = 0;
+		}
+		
+		// Is the buffer full?
+		if (nChars == buffer.length)
+			return 0;
+		
+		int read;
+		do {
+			read = in.read(buffer, nChars, buffer.length - nChars);
+		} while (read == 0);
+		
+		if (read > 0) {
+			nChars += read;
+		}
+		return read;
+	}
+
+	@Override
+	public void close() {
+		FileUtil.close(in);
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/io/RawStreamMuncher.java

@@ -0,0 +1,34 @@
+package com.semmle.util.io;
+
+import com.semmle.util.exception.Exceptions;
+import com.semmle.util.files.FileUtil;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * A thread that copies data from an input stream to an output stream. When
+ * the input stream runs out, it closes both the input and output streams.
+ */
+public class RawStreamMuncher extends Thread {
+	private final InputStream in;
+	private final OutputStream out;
+
+	public RawStreamMuncher(InputStream in, OutputStream out) {
+		this.in = in;
+		this.out = out;
+	}
+
+	@Override
+	public void run() {
+		try {
+			StreamUtil.copy(in, out);
+		} catch (IOException e) {
+			Exceptions.ignore(e, "When the process exits, a harmless IOException will occur here");
+		} finally {
+			FileUtil.close(in);
+			FileUtil.close(out);
+		}
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/io/StreamMuncher.java

@@ -0,0 +1,49 @@
+package com.semmle.util.io;
+
+import com.semmle.util.exception.Exceptions;
+import com.semmle.util.files.FileUtil;
+import com.semmle.util.io.BufferedLineReader;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.PrintStream;
+
+/**
+ * A thread that forwards data from one stream to another. It waits for
+ * entire lines of input from one stream before writing data to the next
+ * stream, and it flushes as it goes.
+ */
+public class StreamMuncher extends Thread {
+	private final InputStream is;
+	private PrintStream output;
+	private BufferedLineReader reader;
+
+	public StreamMuncher(InputStream is, OutputStream output) {
+		this.is = is;
+		if (output != null)
+			this.output = new PrintStream(output);
+	}
+
+	@Override
+	public void run() {
+		InputStreamReader isr = null;
+		try {
+			isr = new InputStreamReader(is);
+			reader = new BufferedLineReader(isr);
+			String line;
+			while ((line = reader.readLineAndTerminator()) != null) {
+				if (output != null) {
+					output.print(line);
+					output.flush();
+				}
+			}
+		} catch (IOException e) {
+			Exceptions.ignore(e, "When the process exits, a harmless IOException will occur here");
+		} finally {
+			FileUtil.close(reader);
+			FileUtil.close(isr);
+		}
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/io/StreamUtil.java

@@ -0,0 +1,201 @@
+package com.semmle.util.io;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Reader;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+
+import com.semmle.util.exception.CatastrophicError;
+
+/**
+ * Utility methods concerning {@link InputStream}s and {@link OutputStream}s.
+ */
+public class StreamUtil
+{
+	/**
+	 * Copy all bytes that can be read from an {@link InputStream}, into an {@link OutputStream}.
+	 *
+	 * @param inputStream The InputStream from which to read, until an
+	 *          {@link InputStream#read(byte[])} operation returns indicating that the input stream
+	 *          has reached its end.
+	 * @param outputStream The OutputStream to which all bytes read from {@code inputStream} should be
+	 *          written.
+	 * @return The number of bytes copied.
+	 * @throws IOException from {@link InputStream#read(byte[])} or
+	 *           {@link OutputStream#write(byte[], int, int)}
+	 * @throws CatastrophicError if either of the streams is {@code null}
+	 */
+	public static long copy(InputStream inputStream, OutputStream outputStream) throws IOException
+	{
+		nullCheck(inputStream, outputStream);
+
+		// Copy byte data
+		long   total = 0;
+		byte[] bytes = new byte[1024];
+		int    read;
+		while ((read = inputStream.read(bytes)) > 0) {
+			outputStream.write(bytes, 0, read);
+			total += read;
+		}
+		return total;
+	}
+	
+	/**
+	 * Copy all chars that can be read from a {@link Reader}, into a {@link Writer}.
+	 *
+	 * @param reader The Reader from which to read, until a {@link Reader#read(char[])} operation
+	 *            returns indicating that the reader has reached its end.
+	 * @param writer The Writer to which all characters read from {@code reader} should be written.
+	 * @return The number of bytes copied.
+	 * @throws IOException from {@link Reader#read(char[])} or
+	 *             {@link Writer#write(char[], int, int)}
+	 * @throws CatastrophicError if either of the streams is {@code null}
+	 */
+	public static long copy(Reader reader, Writer writer) throws IOException
+	{
+		nullCheck(reader, writer);
+
+		// Copy byte data
+		long   total = 0;
+		char[] chars = new char[1024];
+		int    read;
+		while ((read = reader.read(chars)) > 0) {
+			writer.write(chars, 0, read);
+			total += read;
+		}
+		return total;
+	}
+
+	/**
+	 * Copy at most {@code length} bytes from an {@link InputStream}, into an {@link OutputStream}.
+	 * <p>
+	 * Note that this method will busy-wait during periods for which the {@code inputStream} cannot
+	 * supply any data, but has not reached its end.
+	 * </p>
+	 *
+	 * @param inputStream The InputStream from which to read, until {@code length} bytes have
+	 *          been read or {@link InputStream#read(byte[], int, int)} operation returns
+	 *          indicating that the input stream has reached its end.
+	 * @param outputStream The OutputStream to which all bytes read from {@code inputStream} should be
+	 *          written.
+	 * @param length The maximum number of bytes to copy
+	 * @return The number of bytes copied.
+	 * @throws IOException from {@link InputStream#read(byte[], int, int)} or
+	 *           {@link OutputStream#write(byte[], int, int)}
+	 * @throws CatastrophicError if either of the streams is {@code null}
+	 */
+	public static long limitedCopy(InputStream inputStream, OutputStream outputStream, long length) throws IOException
+	{
+		nullCheck(inputStream, outputStream);
+
+		// Copy byte data
+		long   total = 0;
+		byte[] bytes = new byte[1024];
+		int    read;
+		while ((read = inputStream.read(bytes, 0, (int) Math.min(bytes.length, length))) > 0) {
+			outputStream.write(bytes, 0, read);
+			length -= read;
+			total += read;
+		}
+		return total;
+	}
+
+	private static void nullCheck(Object input, Object output) {
+		CatastrophicError.throwIfAnyNull(input, output);
+	}
+
+	/**
+	 * Skips over and discards n bytes of data from an input stream. If n is negative then no bytes are skipped.
+	 * @param stream the InputStream
+	 * @param n the number of bytes to be skipped.
+	 * @return false if the end-of-file was reached before successfully skipping n bytes
+	 */
+	public static boolean skip(InputStream stream, long n) throws IOException {
+		if (n <= 0)
+			return true;
+		long toSkip = n - 1;
+
+		while (toSkip > 0) {
+			long skipped = stream.skip(toSkip);
+			if (skipped == 0) {
+				if(stream.read() == -1)
+					return false;
+				else
+					skipped++;
+			}
+			toSkip -= skipped;
+		}
+		if(stream.read() == -1)
+			return false;
+		else
+			return true;
+	}
+
+	/**
+	 * Reads n bytes from the input stream and returns them. This method will block
+	 * until all n bytes are available. If the end of the stream is reached before n bytes are
+	 * read it returns just the read bytes.
+	 *
+	 * @param stream the InputStream
+	 * @param n the number of bytes to read
+	 * @return the read bytes
+	 * @throws IOException if an IOException occurs when accessing the stream
+	 * @throws IllegalArgumentException if n is negative
+	 */
+	public static byte[] readN(InputStream stream, int n) throws IOException {
+		if (n < 0) throw new IllegalArgumentException("n must be positive");
+
+		ByteArrayOutputStream bOut = new ByteArrayOutputStream();
+		limitedCopy(stream, bOut, n);
+		return bOut.toByteArray();
+	}
+
+	/**
+	 * Reads bytes from the input stream into the given buffer. This method will block
+	 * until all bytes are available. If the end of the stream is reached before enough bytes are
+	 * read it reads as much as it can.
+	 *
+	 * @param stream the InputStream
+	 * @param buf the buffer to read into
+	 * @param offset the offset to read into
+	 * @param length the number of bytes to read
+	 * @return the total number of read bytes
+	 * @throws IOException if an IOException occurs when accessing the stream
+	 * @throws IllegalArgumentException if n is negative
+	 */
+	public static int read(InputStream stream, byte[] buf, int offset, int length) throws IOException {
+		if (length < 0) throw new IllegalArgumentException("length must be positive");
+
+		// Copy byte data
+		int total = 0;
+		int read;
+		while ((read = stream.read(buf, offset, length)) > 0) {
+			length -= read;
+			total += read;
+		}
+
+		return total;
+	}
+
+	/**
+	 * Convenience method for constructing a buffered reader with a UTF8 charset.
+	 */
+	public static BufferedReader newUTF8BufferedReader(InputStream inputStream) {
+		return new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
+	}
+
+	/**
+	 * Convenience method for constructing a buffered writer with a UTF8 charset.
+	 */
+	public static BufferedWriter newUTF8BufferedWriter(OutputStream outputStream) {
+		return new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8));
+	}
+
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/io/WholeIO.java

@@ -0,0 +1,548 @@
+package com.semmle.util.io;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
+import java.io.Writer;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+import java.util.regex.Pattern;
+
+import com.semmle.util.array.ArrayUtil;
+import com.semmle.util.data.IntRef;
+import com.semmle.util.exception.ResourceError;
+import com.semmle.util.files.FileUtil;
+
+/**
+ * A class that allows bulk operations on entire files,
+ * reading or writing them as {@link String} values.
+ * 
+ * This is intended to address the woeful inadequacy of
+ * the Java standard libraries in this area.
+ */
+public class WholeIO {
+	private IOException e;
+	
+	/**
+	 * Regular expression {@link Pattern}
+	 */
+	private final static Pattern rpLineEndingCRLF = Pattern.compile("\r\n");
+	
+	/**
+	 * The default encoding to use for writing, and for reading if no
+	 * encoding can be detected.
+	 */
+	private final String defaultEncoding;
+	
+	/**
+	 * Construct a new {@link WholeIO} instance using ODASA's default
+	 * charset ({@code "UTF-8"}) for all input and output (unless a
+	 * different encoding is detected for a file being read).
+	 */
+	public WholeIO() {
+		this("UTF-8");
+	}
+	
+	/**
+	 * Construct a new {@link WholeIO} instance using the specified
+	 * encoding for all input and output (unless a different encoding
+	 * is detected for a file being read).
+	 * 
+	 * @param encoding The encoding name, e.g. {@code "UTF-8"}.
+	 */
+	public WholeIO(String encoding) {
+		defaultEncoding = encoding;
+	}
+	
+	/**
+	 * Open the given file for reading, get the entire content
+	 * and return it as a {@link String}. Returns <code>null</code>
+	 * on error, in which case you can check the getLastException()
+	 * method for the exception that occurred.
+	 * 
+	 * <b>Warning:</b> This method trims the content of the file, removing
+	 * leading and trailing whitespace. Do not use it if you care about file
+	 * locations being preserved; use 'read' instead.
+	 * 
+	 * @param file The file to read
+	 * @return The <b>trimmed</b> contents of the file, or <code>null</code> on error.
+	 */
+	public String readAndTrim(File file) {
+		e = null;
+		FileInputStream f = null;
+		try {
+			f = new FileInputStream(file);
+			String contents = readString(f);
+			return contents == null ? null : contents.trim();
+		} catch (IOException e) {
+			this.e = e;
+			return null;
+		} finally {
+			FileUtil.close(f);
+		}
+	}
+
+	/**
+	 * Open the given filename for writing and dump the given
+	 * {@link String} into it. Returns <code>false</code>
+	 * on error, in which case you can check the getLastException()
+	 * method for the exception that occurred. Tries to create any
+	 * enclosing directories that do not exist.
+	 * 
+	 * @param filename The name of the file to write to
+	 * @param contents the string to write out
+	 * @return the success state
+	 */
+	public boolean write(String filename, String contents) {
+		return write(new File(filename), contents);
+	}
+
+	/**
+	 * Open the given filename for writing and dump the given
+	 * {@link String} into it. Returns <code>false</code>
+	 * on error, in which case you can check the getLastException()
+	 * method for the exception that occurred. Tries to create any
+	 * enclosing directories that do not exist.
+	 * 
+	 * @param file The file to write to
+	 * @param contents the string to write out
+	 * @return the success state
+	 */
+	public boolean write(File file, String contents) {
+		return write(file, contents, false);
+	}
+
+	/**
+	 * Open the given path for writing and dump the given
+	 * {@link String} into it. Returns <code>false</code>
+	 * on error, in which case you can check the getLastException()
+	 * method for the exception that occurred. Tries to create any
+	 * enclosing directories that do not exist.
+	 *
+	 * @param path The path to write to
+	 * @param contents the string to write out
+	 * @return the success state
+	 */
+	public boolean write(Path path, String contents) {
+		return write(path, contents, false);
+	}
+
+	/**
+	 * Open the given filename for writing and dump the given
+	 * {@link String} into it. Throws {@link ResourceError}
+	 * if we fail.
+	 * 
+	 * @param file The file to write to
+	 * @param contents the string to write out
+	 */
+	public void strictwrite(File file, String contents) {
+		strictwrite(file, contents, false);
+	}
+
+	/**
+	 * Open the given path for writing and dump the given
+	 * {@link String} into it. Throws {@link ResourceError}
+	 * if we fail.
+	 *
+	 * @param path The path to write to
+	 * @param contents the string to write out
+	 */
+	public void strictwrite(Path path, String contents) {
+		strictwrite(path, contents, false);
+	}
+
+	/**
+	 * This is the same as {@link #write(File,String)},
+	 * except that this method allows appending to an existing file.
+	 * 
+	 * @param file the file to write to
+	 * @param contents the string to write out
+	 * @param append whether or not to append to any existing file
+	 * @return the success state
+	 */
+	public boolean write(File file, String contents, boolean append) {
+		if (file.getParentFile() != null)
+			file.getParentFile().mkdirs();
+		
+		FileOutputStream fos = null;
+		try {
+			fos = new FileOutputStream(file, append);
+			Writer writer = new OutputStreamWriter(fos, Charset.forName(defaultEncoding));
+			writer.append(contents);
+			writer.close();
+			return true;
+		} catch (IOException e) {
+			this.e = e;
+			return false;
+		} finally {
+			FileUtil.close(fos);
+		}
+	}
+
+	/**
+	 * This is the same as {@link #write(Path,String)},
+	 * except that this method allows appending to an existing file.
+	 *
+	 * @param path the path to write to
+	 * @param contents the string to write out
+	 * @param append whether or not to append to any existing file
+	 * @return the success state
+	 */
+	public boolean write(Path path, String contents, boolean append) {
+		try {
+			if (path.getParent() != null)
+				Files.createDirectories(path.getParent());
+
+			try (Writer writer = Files.newBufferedWriter(path, Charset.forName(defaultEncoding),
+					StandardOpenOption.CREATE, StandardOpenOption.WRITE,
+					append ? StandardOpenOption.APPEND : StandardOpenOption.TRUNCATE_EXISTING)) {
+				writer.append(contents);
+			}
+		} catch (IOException e) {
+			this.e = e;
+			return false;
+		}
+		return true;
+	}
+
+	/**
+	 * This is the same as {@link #strictwrite(File,String)},
+	 * except that this method allows appending to an existing file.
+	 */
+	public void strictwrite(File file, String contents, boolean append) {
+		if (!write(file, contents, append))
+			throw new ResourceError("Failed to write file " + file, getLastException());
+	}
+
+	/**
+	 * This is the same as {@link #strictwrite(Path,String)},
+	 * except that this method allows appending to an existing file.
+	 */
+	public void strictwrite(Path path, String contents, boolean append) {
+		if (!write(path, contents, append))
+			throw new ResourceError("Failed to write path " + path, getLastException());
+	}
+
+	/**
+	 * Get the exception that occurred during the last call to
+	 * read(), if any. If the last read() call completed normally,
+	 * this returns null.
+	 * @return The last caught exception, or <code>null</code> if N/A.
+	 */
+	public IOException getLastException() {
+		return e;
+	}
+	
+	public String read(File file) {
+		InputStream is = null;
+		try {
+			is = new FileInputStream(file);
+			return readString(is);
+		}
+		catch (IOException e) {
+			this.e = e;
+			return null;
+		}
+		finally {
+			FileUtil.close(is);
+		}
+	}
+
+	public String read(Path path) {
+		InputStream is = null;
+		try {
+			is = Files.newInputStream(path);
+			return readString(is);
+		}
+		catch (IOException e) {
+			this.e = e;
+			return null;
+		}
+		finally {
+			FileUtil.close(is);
+		}
+	}
+
+	/**
+	 * Read the contents of the given {@link File} as text (line endings are normalised to "\n" in the output).
+	 *
+	 * @param file	The file to read.
+	 * @return		The text contents of the file, if possible, or null if the file cannot be read.
+	 */
+	public String readText(File file) {
+		String result = read(file);
+		return result != null ? result.replaceAll("\r\n", "\n") : null;
+	}
+
+	/**
+	 * Read the contents of the given {@link Path} as text (line endings are normalised to "\n" in the output).
+	 *
+	 * @param path	The path to read.
+	 * @return		The text contents of the path, if possible, or null if the file cannot be read.
+	 */
+	public String readText(Path path) {
+		String result = read(path);
+		return result != null ? result.replaceAll("\r\n", "\n") : null;
+	}
+	
+	
+	/**
+	 * Read the contents of the given {@link File}, throwing a {@link ResourceError}
+	 * if we fail.
+	 */
+	public String strictread(File f) {
+		String content = read(f);
+		if (content == null)
+			throw new ResourceError("Failed to read file " + f, getLastException());
+		return content;
+	}
+
+	/**
+	 * Read the contents of the given {@link Path}, throwing a {@link ResourceError}
+	 * if we fail.
+	 */
+	public String strictread(Path f) {
+		String content = read(f);
+		if (content == null)
+			throw new ResourceError("Failed to read path " + f, getLastException());
+		return content;
+	}
+
+	/**
+	 * Read the contents of the given {@link File} as text (line endings are normalised to "\n" in the output).
+	 *
+	 * @param file				The file to read.
+	 * @return					The text contents of the file, if possible.
+	 * @throws ResourceError	If the file cannot be read.
+	 */
+	public String strictreadText(File file) {
+		return rpLineEndingCRLF.matcher(strictread(file)).replaceAll("\n");
+	}
+
+	/**
+	 * Read the contents of the given {@link Path} as text (line endings are normalised to "\n" in the output).
+	 *
+	 * @param path				The path to read.
+	 * @return					The text contents of the path, if possible.
+	 * @throws ResourceError	If the path cannot be read.
+	 */
+	public String strictreadText(Path path) {
+		return rpLineEndingCRLF.matcher(strictread(path)).replaceAll("\n");
+	}
+
+	/**
+	 * Get the entire content of an {@link InputStream}
+	 * and interpret it as a {@link String} trying to detect its character set.
+	 * Returns <code>null</code> on error, in which case you can check
+	 * the getLastException() method for the exception that occurred.
+	 * 
+	 * @param stream the stream to read from
+	 * @return The contents of the file, or <code>null</code> on error.
+	 */
+	public String readString(InputStream stream) {
+		IntRef length = new IntRef(0);
+		byte[] bytes = readBinary(stream, length);
+		
+		if (bytes == null) return null;
+		
+		try {
+			IntRef start = new IntRef(0);
+			String charset = determineCharset(bytes, length.get(), start);
+			return new String(bytes, start.get(), length.get() - start.get(), charset);
+		} catch (UnsupportedEncodingException e) {
+			this.e = e;
+			return null;
+		}
+	}
+
+	/**
+	 * Get the entire content of an {@link InputStream}
+	 * and interpret it as a {@link String} trying to detect its character set.
+	 * Throws a {@link ResourceError} on error.
+	 *
+	 * @param stream the stream to read from
+	 * @return the contents of the input stream
+	 */
+	public String strictReadString(InputStream stream) {
+		String content = readString(stream);
+		if (content == null)
+			throw new ResourceError("Could not read from stream", getLastException());
+		return content;
+	}
+
+	/**
+	 * Get the entire content of an {@link InputStream}, interpreting it
+	 * as a sequence of bytes. This removes restrictions regarding invalid
+	 * code points that would potentially prevent reading a file's contents
+	 * as a String.
+	 * 
+	 * This method returns <code>null</code> on error, in which case you can
+	 * check {@link #getLastException()} for the exception that occurred.
+	 * 
+	 * @param stream the stream to read from
+	 * @return The binary contents of the file, or <code>null</code> on error.
+	 */
+	public byte[] readBinary(InputStream stream) {
+		IntRef length = new IntRef(0);
+		byte[] bytes = readBinary(stream, length);
+		return bytes == null ? null : Arrays.copyOf(bytes, length.get());
+	}
+
+	/**
+	 * Get the entire content of an {@link InputStream}, interpreting it
+	 * as a sequence of bytes. This removes restrictions regarding invalid
+	 * code points that would potentially prevent reading a file's contents
+	 * as a String.
+	 *
+	 * @param stream the stream to read from
+	 * @return The binary contents of the file -- always non-null.
+	 * @throws ResourceError if an exception occurs during IO.
+	 */
+	public byte[] strictReadBinary(InputStream stream) {
+		byte[] result = readBinary(stream);
+		if (result == null)
+			throw new ResourceError("Couldn't read from stream", e);
+		return result;
+	}
+	
+	/**
+	 * Get the entire binary contents of a {@link File} as a sequence of bytes.
+	 * 
+	 * @param file the file to read
+	 * @return the file's contents as a byte[] -- always non-null.
+	 * @throws ResourceError if an exception occurs during IO.
+	 */
+	public byte[] strictReadBinary(File file) {
+		FileInputStream stream = null;
+		try {
+			stream = new FileInputStream(file);
+			byte[] result = readBinary(stream);
+			if (result == null)
+				throw new ResourceError("Couldn't read from file " + file + ".", e);
+			return result;
+		} catch (FileNotFoundException e) {
+			throw new ResourceError("Couldn't read from file " + file + ".", e);
+		} finally {
+			FileUtil.close(stream);
+		}
+	}
+
+	/**
+	 * Get the entire binary contents of a {@link Path} as a sequence of bytes.
+	 * 
+	 * @param path the path to read
+	 * @return the file's contents as a byte[] -- always non-null.
+	 * @throws ResourceError if an exception occurs during IO.
+	 */
+	public byte[] strictReadBinary(Path path) {
+		InputStream stream = null;
+		try {
+			stream = Files.newInputStream(path);
+			byte[] result = readBinary(stream);
+			if (result == null)
+				throw new ResourceError("Couldn't read from path " + path + ".", e);
+			return result;
+		} catch (IOException e) {
+			throw new ResourceError("Couldn't read from path " + path + ".", e);
+		} finally {
+			FileUtil.close(stream);
+		}
+	}
+
+	/**
+	 * Get the entire binary contents of a {@link Path} as a sequence of bytes.
+	 * 
+	 * @param path the path to read
+	 * @return the file's contents as a byte[] -- always non-null.
+	 */
+	public byte[] readBinary(Path path) throws IOException {
+		InputStream stream = null;
+		try {
+			stream = Files.newInputStream(path);
+			byte[] result = readBinary(stream);
+			if (result == null)
+				throw new ResourceError("Couldn't read from path " + path + ".", e);
+			return result;
+		} finally {
+			FileUtil.close(stream);
+		}
+	}
+	
+	private byte[] readBinary(InputStream stream, IntRef offsetHolder) {
+		try {
+			byte[] bytes = new byte[16384];
+			int offset = 0;
+			int readThisTime;
+			do {
+				readThisTime = stream.read(bytes, offset, bytes.length - offset);
+				if (readThisTime > 0) {
+					offset += readThisTime;
+					if (offset == bytes.length)
+						bytes = safeArrayDouble(bytes);
+				}
+			} while (readThisTime > 0);
+			offsetHolder.set(offset);
+			return bytes;
+		} catch (IOException e) {
+			this.e = e;
+			return null;
+		}
+	}
+
+	/**
+	 * Safely attempt to double the length of an array.
+	 * @param array The array which want to be doubled
+	 * @return a new array that is longer than array 
+	 */
+	private byte[] safeArrayDouble(byte[] array) {
+		if (array.length  >= ArrayUtil.MAX_ARRAY_LENGTH) {
+			throw new ResourceError("Cannot stream into array as it exceed the maximum array size");
+		}
+		// Compute desired capacity
+		long newCapacity = array.length * 2L;
+		// Ensure it is at least as large as minCapacity
+		if (newCapacity < 16)
+			newCapacity = 16;
+		// Ensure it is at most MAX_ARRAY_LENGTH
+		if (newCapacity > ArrayUtil.MAX_ARRAY_LENGTH) {
+			newCapacity = ArrayUtil.MAX_ARRAY_LENGTH;
+		}
+		return Arrays.copyOf(array, (int)newCapacity);
+	}
+
+	/**
+	 * Try to determine the encoding of a byte[] using a byte-order mark (if present)
+	 * Defaults to UTF-8 if none found.
+	 */
+	private String determineCharset(byte[] bom, int length, IntRef start) {
+		start.set(0);
+		String ret = defaultEncoding;
+		if(length < 2)
+			return ret;
+		if (length >= 3 && byteToInt(bom[0]) == 0xEF && byteToInt(bom[1]) == 0xBB && byteToInt(bom[2]) == 0xBF) {
+			ret = "UTF-8";
+			start.set(3);
+		} else if (byteToInt(bom[0]) == 0xFE && byteToInt(bom[1]) == 0xFF) {
+			ret = "UTF-16BE";
+			start.set(2);
+		} else if (byteToInt(bom[0]) == 0xFF && byteToInt(bom[1]) == 0xFE) {
+			ret = "UTF-16LE";
+			start.set(2);
+		}
+		return ret;
+	}
+	
+	private static int byteToInt(byte b) {
+		return b & 0xFF;
+	}
+
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVParser.java

@@ -0,0 +1,207 @@
+package com.semmle.util.io.csv;
+
+/**
+ Copyright 2005 Bytecode Pty Ltd.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A very simple CSV parser released under a commercial-friendly license.
+ * This just implements splitting a single line into fields.
+ *
+ * @author Glen Smith
+ * @author Rainer Pruy
+ *
+ */
+public class CSVParser {
+
+	private final char separator;
+
+	private final char quotechar;
+
+	private final char escape;
+
+	private final boolean strictQuotes;
+
+	private StringBuilder buf = new StringBuilder(INITIAL_READ_SIZE);
+
+	/** The default separator to use if none is supplied to the constructor. */
+	public static final char DEFAULT_SEPARATOR = ',';
+
+	private static final int INITIAL_READ_SIZE = 128;
+
+	/**
+	 * The default quote character to use if none is supplied to the
+	 * constructor.
+	 */
+	public static final char DEFAULT_QUOTE_CHARACTER = '"';
+
+
+	/**
+	 * The default escape character to use if none is supplied to the
+	 * constructor.
+	 */
+	public static final char DEFAULT_ESCAPE_CHARACTER = '"';
+
+	/**
+	 * The default strict quote behavior to use if none is supplied to the
+	 * constructor
+	 */
+	public static final boolean DEFAULT_STRICT_QUOTES = false;
+
+	/**
+	 * Constructs CSVReader with supplied separator and quote char.
+	 * Allows setting the "strict quotes" flag
+	 * @param separator
+	 *            the delimiter to use for separating entries
+	 * @param quotechar
+	 *            the character to use for quoted elements
+	 * @param escape
+	 *            the character to use for escaping a separator or quote
+	 * @param strictQuotes
+	 *            if true, characters outside the quotes are ignored
+	 */
+	CSVParser(char separator, char quotechar, char escape, boolean strictQuotes) {
+		this.separator = separator;
+		this.quotechar = quotechar;
+		this.escape = escape;
+		this.strictQuotes = strictQuotes;
+	}
+
+	/**
+	 *
+	 * @return true if something was left over from last call(s)
+	 */
+	public boolean isPending() {
+		return buf.length() != 0;
+	}
+
+	public String[] parseLineMulti(String nextLine) throws IOException {
+		return parseLine(nextLine, true);
+	}
+
+	public String[]  parseLine(String nextLine) throws IOException {
+		return parseLine(nextLine, false);
+	}
+	/**
+	 * Parses an incoming String and returns an array of elements.
+	 *
+	 * @param nextLine
+	 *            the string to parse
+	 * @return the comma-tokenized list of elements, or null if nextLine is null
+	 * @throws IOException if bad things happen during the read
+	 */
+	private String[] parseLine(String nextLine, boolean multi) throws IOException {
+
+		if (!multi && isPending()) {
+			clear();
+		}
+
+		if (nextLine == null) {
+			if (isPending()) {
+				String s = buf.toString();
+				clear();
+				return new String[] {s};
+			} else {
+				return null;
+			}
+		}
+
+		List<String>tokensOnThisLine = new ArrayList<String>();
+		boolean inQuotes = isPending();
+		for (int i = 0; i < nextLine.length(); i++) {
+
+			char c = nextLine.charAt(i);
+			if (c == this.escape && isNextCharacterEscapable(nextLine, inQuotes, i)) {
+				buf.append(nextLine.charAt(i+1));
+				i++;
+			} else if (c == quotechar) {
+				if( isNextCharacterEscapedQuote(nextLine, inQuotes, i) ){
+					buf.append(nextLine.charAt(i+1));
+					i++;
+				}else{
+					inQuotes = !inQuotes;
+					// the tricky case of an embedded quote in the middle: a,bc"d"ef,g
+					if (!strictQuotes) {
+						if(i>2 //not on the beginning of the line
+								&& nextLine.charAt(i-1) != this.separator //not at the beginning of an escape sequence
+								&& nextLine.length()>(i+1) &&
+								nextLine.charAt(i+1) != this.separator //not at the	end of an escape sequence
+								){
+							buf.append(c);
+						}
+					}
+				}
+			} else if (c == separator && !inQuotes) {
+				tokensOnThisLine.add(buf.toString());
+				clear(); // start work on next token
+			} else {
+				if (!strictQuotes || inQuotes)
+					buf.append(c);
+			}
+		}
+		// line is done - check status
+		if (inQuotes) {
+			if (multi) {
+				// continuing a quoted section, re-append newline
+				buf.append('\n');
+				// this partial content is not to be added to field list yet
+			} else {
+				throw new IOException("Un-terminated quoted field at end of CSV line");
+			}
+		} else {
+			tokensOnThisLine.add(buf.toString());
+			clear();
+		}
+		return tokensOnThisLine.toArray(new String[tokensOnThisLine.size()]);
+
+	}
+
+	/**
+	 * precondition: the current character is a quote or an escape
+	 * @param nextLine the current line
+	 * @param inQuotes true if the current context is quoted
+	 * @param i current index in line
+	 * @return true if the following character is a quote
+	 */
+	private boolean isNextCharacterEscapedQuote(String nextLine, boolean inQuotes, int i) {
+		return inQuotes  // we are in quotes, therefore there can be escaped quotes in here.
+				&& nextLine.length() > (i+1)  // there is indeed another character to check.
+				&& nextLine.charAt(i+1) == quotechar;
+	}
+
+	/**
+	 * precondition: the current character is an escape
+	 * @param nextLine the current line
+	 * @param inQuotes true if the current context is quoted
+	 * @param i current index in line
+	 * @return true if the following character is a quote
+	 */
+	protected boolean isNextCharacterEscapable(String nextLine, boolean inQuotes, int i) {
+		return inQuotes  // we are in quotes, therefore there can be escaped quotes in here.
+				&& nextLine.length() > (i+1)  // there is indeed another character to check.
+				&& ( nextLine.charAt(i+1) == quotechar || nextLine.charAt(i+1) == this.escape);
+	}
+
+	/**
+	 * Reset the buffer used for storing the current field's value
+	 */
+	private void clear() {
+		buf.setLength(0);
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVReader.java

@@ -0,0 +1,192 @@
+package com.semmle.util.io.csv;
+
+/**
+ Copyright 2005 Bytecode Pty Ltd.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+import java.io.BufferedReader;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A very simple CSV reader released under a commercial-friendly license.
+ * 
+ * @author Glen Smith
+ * 
+ */
+public class CSVReader implements Closeable {
+
+    private final BufferedReader br;
+
+    private boolean hasNext = true;
+
+    private final CSVParser parser;
+    
+    private final int skipLines;
+
+    private boolean linesSkipped;
+
+    /** The line number of the last physical line read (one-based). */
+    private int curline = 0;
+
+    /** The physical line number at which the last logical line read started (one-based). */
+    private int startLine = 0;
+
+    /**
+     * The default line to start reading.
+     */
+    private static final int DEFAULT_SKIP_LINES = 0;
+
+    /**
+     * Constructs CSVReader using a comma for the separator.
+     * 
+     * @param reader
+     *            the reader to an underlying CSV source.
+     */
+    public CSVReader(Reader reader) {
+        this(reader,
+            CSVParser.DEFAULT_SEPARATOR, CSVParser.DEFAULT_QUOTE_CHARACTER,
+            CSVParser.DEFAULT_ESCAPE_CHARACTER, DEFAULT_SKIP_LINES,
+            CSVParser.DEFAULT_STRICT_QUOTES);
+    }
+
+    /**
+     * Constructs CSVReader with supplied separator and quote char.
+     * 
+     * @param reader
+     *            the reader to an underlying CSV source.
+     * @param separator
+     *            the delimiter to use for separating entries
+     * @param quotechar
+     *            the character to use for quoted elements
+     * @param escape
+     *            the character to use for escaping a separator or quote
+     * @param line
+     *            the line number to skip for start reading
+     * @param strictQuotes
+     *            sets if characters outside the quotes are ignored
+     */
+    private CSVReader(Reader reader, char separator, char quotechar, char escape, int line, boolean strictQuotes) {
+        this.br = new BufferedReader(reader);
+        this.parser = new CSVParser(separator, quotechar, escape, strictQuotes);
+        this.skipLines = line;
+    }
+
+
+	/**
+     * Reads the entire file into a List with each element being a String[] of
+     * tokens.
+     * 
+     * @return a List of String[], with each String[] representing a line of the
+     *         file.
+     * 
+     * @throws IOException
+     *             if bad things happen during the read
+     */
+    public List<String[]> readAll() throws IOException {
+
+        List<String[]> allElements = new ArrayList<String[]>();
+        while (hasNext) {
+            String[] nextLineAsTokens = readNext();
+            if (nextLineAsTokens != null)
+                allElements.add(nextLineAsTokens);
+        }
+        return allElements;
+
+    }
+
+    /**
+     * Reads the next line from the buffer and converts to a string array.
+     * 
+     * @return a string array with each comma-separated element as a separate
+     *         entry, or null if there are no more lines to read.
+     * 
+     * @throws IOException
+     *             if bad things happen during the read
+     */
+    public String[] readNext() throws IOException {
+    	boolean first = true;
+    	String[] result = null;
+    	do {
+    		String nextLine = getNextLine();
+
+    		if (first) {
+    			startLine = curline;
+    			first = false;
+    		}
+
+    		if (!hasNext) {
+    			return result; // should throw if still pending?
+    		}
+    		String[] r = parser.parseLineMulti(nextLine);
+    		if (r.length > 0) {
+    			if (result == null) {
+    				result = r;
+    			} else {
+    				String[] t = new String[result.length+r.length];
+    				System.arraycopy(result, 0, t, 0, result.length);
+    				System.arraycopy(r, 0, t, result.length, r.length);
+    				result = t;
+    			}
+    		}
+    	} while (parser.isPending());
+    	return result;
+    }
+
+    /**
+     * Reads the next line from the file.
+     * 
+     * @return the next line from the file without trailing newline
+     * @throws IOException
+     *             if bad things happen during the read
+     */
+    private String getNextLine() throws IOException {
+    	if (!this.linesSkipped) {
+            for (int i = 0; i < skipLines; i++) {
+                br.readLine();
+                ++curline;
+            }
+            this.linesSkipped = true;
+        }
+        String nextLine = br.readLine();
+        if (nextLine == null) {
+            hasNext = false;
+        } else {
+            ++curline;
+        }
+        return hasNext ? nextLine : null;
+    }
+
+    /**
+     * Closes the underlying reader.
+     * 
+     * @throws IOException if the close fails
+     */
+    @Override
+	public void close() throws IOException{
+    	br.close();
+    }
+
+    /**
+     * Return the physical line number (one-based) at which the last logical line read started,
+     * or zero if no line has been read yet.
+     */
+    public int getStartLine() {
+    	return startLine;
+    }
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVWriter.java

@@ -0,0 +1,226 @@
+package com.semmle.util.io.csv;
+
+/**
+ Copyright 2005 Bytecode Pty Ltd.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.List;
+
+/**
+ * A very simple CSV writer released under a commercial-friendly license.
+ *
+ * @author Glen Smith
+ *
+ */
+public class CSVWriter implements Closeable {
+
+	private static final int INITIAL_STRING_SIZE = 128;
+
+	private Writer rawWriter;
+
+	private char separator;
+
+	private char quotechar;
+
+	private char escapechar;
+
+	private String lineEnd;
+
+	/** The quote constant to use when you wish to suppress all quoting. */
+	public static final char NO_QUOTE_CHARACTER = '\u0000';
+
+	/** The escape constant to use when you wish to suppress all escaping. */
+	private static final char NO_ESCAPE_CHARACTER = '\u0000';
+
+	/** Default line terminator uses platform encoding. */
+	private static final String DEFAULT_LINE_END = "\n";
+
+	private boolean[] eagerQuotingFlags = {};
+
+	/**
+	 * Constructs CSVWriter using a comma for the separator.
+	 *
+	 * @param writer
+	 *            the writer to an underlying CSV source.
+	 */
+	public CSVWriter(Writer writer) {
+		this(writer,
+				CSVParser.DEFAULT_SEPARATOR,
+				CSVParser.DEFAULT_QUOTE_CHARACTER,
+				CSVParser.DEFAULT_ESCAPE_CHARACTER
+				);
+	}
+
+	/**
+	 * Constructs CSVWriter with supplied separator and quote char.
+	 *
+	 * @param writer
+	 *            the writer to an underlying CSV source.
+	 * @param separator
+	 *            the delimiter to use for separating entries
+	 * @param quotechar
+	 *            the character to use for quoted elements
+	 * @param escapechar
+	 *            the character to use for escaping quotechars or escapechars
+	 */
+	public CSVWriter(Writer writer, char separator, char quotechar, char escapechar) {
+		this(writer, separator, quotechar, escapechar, DEFAULT_LINE_END);
+	}
+
+	/**
+	 * Constructs CSVWriter with supplied separator, quote char, escape char and line ending.
+	 *
+	 * @param writer
+	 *            the writer to an underlying CSV source.
+	 * @param separator
+	 *            the delimiter to use for separating entries
+	 * @param quotechar
+	 *            the character to use for quoted elements
+	 * @param escapechar
+	 *            the character to use for escaping quotechars or escapechars
+	 * @param lineEnd
+	 * 			  the line feed terminator to use
+	 */
+	private CSVWriter(Writer writer, char separator, char quotechar, char escapechar, String lineEnd) {
+		this.rawWriter = writer;
+		this.separator = separator;
+		this.quotechar = quotechar;
+		this.escapechar = escapechar;
+		this.lineEnd = lineEnd;
+	}
+
+	/**
+	 * Call with an array of booleans, corresponding to columns, where columns that have
+	 * <code>false</code> will not be quoted unless they contain special characters.
+	 * <p>
+	 * If there are more columns to print than have been configured here, any additional
+	 * columns will be treated as if <code>true</code> was passed.
+	 */
+	public void setEagerQuotingColumns(boolean... flags) {
+		eagerQuotingFlags = flags;
+	}
+
+	/**
+	 * Writes the entire list to a CSV file. The list is assumed to be a
+	 * String[]
+	 *
+	 * @param allLines
+	 *            a List of String[], with each String[] representing a line of
+	 *            the file.
+	 */
+	public void writeAll(List<String[]> allLines) throws IOException  {
+		for (String[] line : allLines) {
+			writeNext(line);
+		}
+	}
+
+	/**
+	 * Writes the next line to the file.
+	 *
+	 * @param nextLine
+	 *            a string array with each comma-separated element as a separate
+	 *            entry.
+	 */
+	public void writeNext(String... nextLine) throws IOException {
+
+		if (nextLine == null)
+			return;
+
+		StringBuilder sb = new StringBuilder(INITIAL_STRING_SIZE);
+		for (int i = 0; i < nextLine.length; i++) {
+
+			if (i != 0) {
+				sb.append(separator);
+			}
+
+			String nextElement = nextLine[i];
+			if (nextElement == null)
+				continue;
+			boolean hasSpecials = stringContainsSpecialCharacters(nextElement);
+
+			if (hasSpecials || i >= eagerQuotingFlags.length || eagerQuotingFlags[i]
+					|| stringContainsSomewhatSpecialCharacter(nextElement)) {
+				if (quotechar != NO_QUOTE_CHARACTER)
+					sb.append(quotechar);
+				sb.append(hasSpecials ? processLine(nextElement) : nextElement);
+				if (quotechar != NO_QUOTE_CHARACTER)
+					sb.append(quotechar);
+			} else {
+				sb.append(nextElement);
+			}
+		}
+
+		sb.append(lineEnd);
+		rawWriter.write(sb.toString());
+
+	}
+
+	/**
+	 * Return true if there are characters that need to be escaped in addition to
+	 * being quoted.
+	 */
+	private boolean stringContainsSpecialCharacters(String line) {
+		return line.indexOf(quotechar) != -1 || line.indexOf(escapechar) != -1;
+	}
+
+	/**
+	 * Return true if there are characters that should not appear in a completely
+	 * unquoted field.
+	 */
+	private boolean stringContainsSomewhatSpecialCharacter(String s) {
+		return s.indexOf('"') != -1 || s.indexOf('\'') != -1 || s.indexOf('\t') != -1 || s.indexOf(separator) != -1;
+	}
+
+	protected StringBuilder processLine(String nextElement)
+	{
+		StringBuilder sb = new StringBuilder(INITIAL_STRING_SIZE);
+		for (int j = 0; j < nextElement.length(); j++) {
+			char nextChar = nextElement.charAt(j);
+			if (escapechar != NO_ESCAPE_CHARACTER && nextChar == quotechar) {
+				sb.append(escapechar).append(nextChar);
+			} else if (escapechar != NO_ESCAPE_CHARACTER && nextChar == escapechar) {
+				sb.append(escapechar).append(nextChar);
+			} else {
+				sb.append(nextChar);
+			}
+		}
+
+		return sb;
+	}
+
+	/**
+	 * Flush underlying stream to writer.
+	 *
+	 * @throws IOException if bad things happen
+	 */
+	public void flush() throws IOException {
+		rawWriter.flush();
+	}
+
+	/**
+	 * Close the underlying stream writer flushing any buffered content.
+	 *
+	 * @throws IOException if bad things happen
+	 *
+	 */
+	@Override
+	public void close() throws IOException {
+	  rawWriter.close();
+	}
+
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/logging/Streams.java

@@ -0,0 +1,101 @@
+package com.semmle.util.logging;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.Stack;
+
+import com.semmle.util.exception.CatastrophicError;
+
+/**
+ * A class to wrap around accesses to {@link System#out} and
+ * {@link System#err}, so that tools can behave consistently when
+ * run in-process or out-of-process.
+ */
+public class Streams {
+	private static final InheritableThreadLocal<PrintStream> out =
+			new InheritableThreadLocal<PrintStream>() {
+		@Override
+		protected PrintStream initialValue() {
+			return System.out;
+		}
+	};
+	
+	private static final InheritableThreadLocal<PrintStream> err = 
+			new InheritableThreadLocal<PrintStream>() {
+		@Override
+		protected PrintStream initialValue() {
+			return System.err;
+		}
+	};
+
+	private static final InheritableThreadLocal<InputStream> in = 
+			new InheritableThreadLocal<InputStream>() {
+		@Override
+		protected InputStream initialValue() {
+			return System.in;
+		}
+	};
+	
+	private static class SavedContext {
+		public PrintStream out, err;
+		public InputStream in;
+	}
+	
+	private static final ThreadLocal<Stack<SavedContext>> contexts = 
+			new ThreadLocal<Stack<SavedContext>>() {
+		@Override
+		protected Stack<SavedContext> initialValue() {
+			return new Stack<SavedContext>();
+		}
+	};
+	
+	public static PrintStream out() {
+		return out.get();
+	}
+	
+	public static PrintStream err() {
+		return err.get();
+	}
+
+	public static InputStream in() {
+		return in.get();
+	}
+
+	public static void pushContext(OutputStream stdout, OutputStream stderr, InputStream stdin) {
+		SavedContext context = new SavedContext();
+		context.out = out.get();
+		context.err = err.get();
+		context.in = in.get();
+		// When we run in-process, we don't benefit from
+		// a clean slate like we do when starting a new
+		// process. We need to reset anything that we care
+		// about manually.
+		// In particular, the parent VM may well have set
+		// showAllLogs=True, and we don't want the extra
+		// noise when executing the child, so we set a
+		// fresh log state for the duration of the child.
+		
+		contexts.get().push(context);
+		out.set(asPrintStream(stdout));
+		err.set(asPrintStream(stderr));
+		in.set(stdin);
+	}
+
+	private static PrintStream asPrintStream(OutputStream stdout) {
+		return stdout instanceof PrintStream ?
+				(PrintStream)stdout : new PrintStream(stdout);
+	}
+	
+	public static void popContext() {
+		Stack<SavedContext> context = contexts.get();
+		out.get().flush();
+		err.get().flush();
+		if (context.isEmpty())
+			throw new CatastrophicError("Popping logging context without preceding push.");
+		SavedContext old = context.pop();
+		out.set(old.out);
+		err.set(old.err);
+		in.set(old.in);
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/process/AbstractProcessBuilder.java

@@ -0,0 +1,398 @@
+package com.semmle.util.process;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import com.github.codeql.Logger;
+import com.github.codeql.Severity;
+
+import com.semmle.util.data.StringUtil;
+import com.semmle.util.exception.CatastrophicError;
+import com.semmle.util.exception.Exceptions;
+import com.semmle.util.exception.InterruptedError;
+import com.semmle.util.exception.ResourceError;
+import com.semmle.util.files.FileUtil;
+import com.semmle.util.io.RawStreamMuncher;
+
+/**
+ * A builder for an external process. This class wraps {@link ProcessBuilder},
+ * adding support for spawning threads to manage the input and output streams of
+ * the created process.
+ */
+public abstract class AbstractProcessBuilder {
+	public static Logger logger = null;
+
+	// timeout for the muncher threads in seconds
+	protected static final long MUNCH_TIMEOUT = 20;
+	private final ProcessBuilder builder;
+	private boolean logFailure = true;
+	private InputStream in;
+	private LeakPrevention leakPrevention;
+
+	private volatile boolean interrupted = false;
+	private volatile Thread threadToInterrupt = null;
+
+	private volatile boolean hitTimeout = false;
+
+	private final Map<String, String> canonicalEnvVarNames = new LinkedHashMap<>();
+
+	private RawStreamMuncher inMuncher;
+
+
+	public AbstractProcessBuilder (List<String> args, File cwd, Map<String, String> env)
+	{
+		// Sanity checks
+		CatastrophicError.throwIfNull(args);
+		for (int i = 0; i < args.size(); ++i)
+			CatastrophicError.throwIfNull(args.get(i));
+
+		leakPrevention = LeakPrevention.NONE;
+		builder = new ProcessBuilder(new ArrayList<>(args));
+		if (cwd != null) {
+			builder.directory(cwd);
+		}
+		// Make sure that values that have been explicitly removed from Env.systemEnv()
+		// -- such as the variables representing command-line arguments --
+		// are not taken over by the new ProcessBuilder.
+		Map<String, String> keepThese = Env.systemEnv().getenv();
+		for (Iterator<String> it = builder.environment().keySet().iterator(); it.hasNext();) {
+			String name = it.next();
+			if (!keepThese.containsKey(name))
+				it.remove();
+		}
+		if (env != null) {
+			addEnvironment(env);
+		}
+
+	}
+
+	public void setLeakPrevention(LeakPrevention leakPrevention) {
+		CatastrophicError.throwIfNull(leakPrevention);
+		this.leakPrevention = leakPrevention;
+	}
+
+	/**
+	 * See {@link ProcessBuilder#redirectErrorStream(boolean)}.
+	 */
+	public void setRedirectErrorStream(boolean redirectErrorStream) {
+		this.builder.redirectErrorStream(redirectErrorStream);
+	}
+
+	public final boolean hasEnvVar(String name) {
+		return builder.environment().containsKey(getCanonicalVarName(name));
+	}
+
+	/**
+	 * Add the specified key/value pair to the environment of the builder,
+	 * overriding any previous environment entry of that name. This method
+	 * provides additional logic to handle systems where environment
+	 * variable names are case-insensitive, ensuring the last-added value
+	 * for a name ends up in the final environment regardless of case.
+	 * @param name The name of the environment variable. Whether case matters
+	 * 		is OS-dependent.
+	 * @param value The value for the environment variable.
+	 */
+	public final void addEnvVar(String name, String value) {
+		builder.environment().put(getCanonicalVarName(name), value);
+	}
+
+	/**
+	 * Prepend a specified set of arguments to this process builder's command line.
+	 * This only makes sense before the builder is started.
+	 */
+	public void prependArgs(List<String> args) {
+		builder.command().addAll(0, args);
+	}
+
+	/**
+	 * Compute a canonical environment variable name relative to this process
+	 * builder.
+	 *
+	 * The need for this method arises on platforms where the environment is
+	 * case-insensitive -- any inspection of it in such a situation needs to
+	 * canonicalise the variable name to have well-defined behaviour. This is
+	 * builder-specific, because it depends on its existing environment. For
+	 * example, if it already defines a variable called <code>Path</code>, and the
+	 * environment is case-insensitive, then setting a variable called
+	 * <code>PATH</code> should overwrite this, and checking whether a variable
+	 * called <code>PATH</code> is already defined should return <code>true</code>.
+	 */
+	public String getCanonicalVarName(String name) {
+		if (!Env.getOS().isEnvironmentCaseSensitive()) {
+			// We need to canonicalise the variable name to work around Java API limitations.
+			if (canonicalEnvVarNames.isEmpty())
+				for (String var : builder.environment().keySet())
+					canonicalEnvVarNames.put(StringUtil.lc(var), var);
+			String canonical = canonicalEnvVarNames.get(StringUtil.lc(name));
+			if (canonical == null)
+				canonicalEnvVarNames.put(StringUtil.lc(name), name);
+			else
+				name = canonical;
+		}
+		return name;
+	}
+
+	/**
+	 * Get a snapshot of this builder's environment, using canonical variable names
+	 * (as per {@link #getCanonicalVarName(String)}) as keys. Modifications to this
+	 * map do not propagate back to the builder; use
+	 * {@link #addEnvVar(String, String)} or {@link #addEnvironment(Map)} to extend
+	 * its environment.
+	 */
+	public Map<String, String> getCanonicalCurrentEnv() {
+		Map<String, String> result = new LinkedHashMap<>();
+		for (Entry<String, String> e : builder.environment().entrySet())
+			result.put(getCanonicalVarName(e.getKey()), e.getValue());
+		return result;
+	}
+
+	/**
+	 * Specify an input stream of data that will be piped to the process's
+	 * standard input.
+	 *
+	 * CAUTION: if this stream is the current process' standard in and no
+	 * input is ever received, then we will leak an uninterruptible thread
+	 * waiting for some input. This will terminate only when the standard in
+	 * is closed, i.e. when the current process terminates.
+	 */
+	public final void setIn(InputStream in) {
+		this.in = in;
+	}
+
+	/**
+	 * Set the environment of this builder to the given map. Any
+	 * existing environment entries (either from the current process
+	 * environment or from previous calls to {@link #addEnvVar(String, String)},
+	 * {@link #addEnvironment(Map)} or {@link #setEnvironment(Map)})
+	 * are discarded.
+	 * @param env The environment to use.
+	 */
+	public final void setEnvironment(Map<String, String> env) {
+		builder.environment().clear();
+		canonicalEnvVarNames.clear();
+		addEnvironment(env);
+	}
+
+	/**
+	 * Add the specified set of environment variables to the environment for
+	 * the builder. This leaves existing variable definitions in place, but
+	 * can override them.
+	 * @param env The environment to merge into the current environment.
+	 */
+	public final void addEnvironment(Map<String, String> env) {
+		for (Entry<String, String> entry : env.entrySet())
+			addEnvVar(entry.getKey(), entry.getValue());
+	}
+
+	public final int execute() {
+		return execute(0);
+	}
+
+	/**
+	 * Set the flag indicating that a non-zero exit code may be expected. This
+	 * will suppress the log of failed commands.
+	 */
+	public final void expectFailure() {
+		logFailure = false;
+	}
+
+	public final int execute(long timeout) {
+		Process process = null;
+		boolean processStopped = true;
+		Timer timer = null;
+		try {
+			synchronized (this) {
+				// Handle the case where we called kill() too early to use
+				// Thread.interrupt()
+				if (interrupted)
+					throw new InterruptedException();
+				threadToInterrupt = Thread.currentThread();
+			}
+
+			processStopped = false;
+			String directory;
+			if (builder.directory() == null) {
+				directory = "current directory ('" + System.getProperty("user.dir") + "')";
+			} else {
+				directory = "'" + builder.directory().toString() + "'";
+			}
+			logger.debug("Running command: '" + toString() + "' in " + directory);
+			process = builder.start();
+			setupInputHandling(process.getOutputStream());
+			setupOutputHandling(process.getInputStream(),
+					process.getErrorStream());
+			if (timeout != 0) {
+				// create the timer's thread as a "daemon" thread, so it does not
+				// prevent the jvm from terminating
+				timer = new Timer(true);
+				final Thread current = Thread.currentThread();
+				timer.schedule(new TimerTask() {
+					@Override
+					public void run() {
+						hitTimeout = true;
+						current.interrupt();
+					}
+				}, timeout);
+			}
+
+			int result = process.waitFor();
+			processStopped = true;
+			if (result != 0 && logFailure)
+				logger.error("Spawned process exited abnormally (code " + result
+						+ "; tried to run: " + getBuilderCommand() + ")");
+			return result;
+		} catch (IOException e) {
+			throw new ResourceError(
+					"IOException while executing process with args: "
+							+ getBuilderCommand(), e);
+		} catch (InterruptedException e) {
+			throw new InterruptedError(
+					"InterruptedException while executing process with args: "
+							+ getBuilderCommand(), e);
+		} finally {
+			// cancel the timer
+			if (timer != null) {
+				timer.cancel();
+			}
+			// clear the interrupted flag of the current thread
+			// in case it was set earlier (ie by the Timer or a call to kill())
+			synchronized (this) {
+				threadToInterrupt = null;
+				Thread.interrupted();
+			}
+			// get rid of the process, in case it is still running.
+			if (process != null && !processStopped) {
+				killProcess(process);
+			}
+			try {
+				cleanupInputHandling();
+				cleanupOutputHandling();
+			} finally {
+
+				if (process != null) {
+					FileUtil.close(process.getErrorStream());
+					FileUtil.close(process.getInputStream());
+					FileUtil.close(process.getOutputStream());
+				}
+			}
+		}
+	}
+
+	/**
+	 * Provides the implementation of actually stopping the child
+	 * process. Provided as an extension point so that this can
+	 * be customised for later Java versions or for other reasons.
+	 */
+	protected void killProcess(Process process) {
+		process.destroy();
+	}
+
+	/**
+	 * Setup handling of the process input stream (stdin).
+	 *
+	 * @param outputStream OutputStream connected to the process's standard input.
+	 */
+	protected void setupInputHandling(OutputStream outputStream) {
+		if (in == null) {
+			FileUtil.close(outputStream);
+			return;
+		}
+		inMuncher = new RawStreamMuncher(in, outputStream);
+		inMuncher.start();
+	}
+
+	/**
+	 * Setup handling of the process' output streams (stdout and stderr).
+	 *
+	 * @param stdout
+	 *            InputStream connected to the process' standard output stream.
+	 * @param stderr
+	 *            InputStream connected to the process' standard error stream.
+	 */
+	protected abstract void setupOutputHandling(InputStream stdout, InputStream stderr);
+
+	/**
+	 * Cleanup resources related to output handling. The method is always called, either after the process
+	 * has exited normally, or after an abnormal termination due to an exception. As a result cleanupOutputHandling()
+	 * might be called, without a previous call to setupOutputHandling. The implementation of this method should
+	 * handle this case.
+	 */
+	protected abstract void cleanupOutputHandling();
+
+	private void cleanupInputHandling() {
+		if (inMuncher != null && inMuncher.isAlive()) {
+			// There's no real need to wait for the muncher to terminate -- on the contrary,
+			// if it's still alive it will typically be waiting for a closing action that
+			// will only happen after execute() returns anyway.
+			// The best we can do is try to interrupt it.
+			inMuncher.interrupt();
+		}
+	}
+
+	protected void waitForMuncher(String which, Thread muncher, long timeout) {
+		// wait for termination of the muncher until a deadline is reached
+		try {
+			muncher.join(timeout);
+
+		} catch (InterruptedException e) {
+			Exceptions.ignore(e,"Further interruption attempts are ineffective --"
+							   + " we're already waiting for termination.");
+		}
+		// if muncher is still alive, report an error
+		if(muncher.isAlive()){
+			muncher.interrupt();
+			logger.error(String.format("Standard %s stream hasn't closed %s seconds after termination of subprocess '%s'.", which, MUNCH_TIMEOUT, this));
+		}
+	}
+
+	public final void kill() {
+		synchronized (this) {
+			interrupted = true;
+			if (threadToInterrupt != null)
+				threadToInterrupt.interrupt();
+		}
+	}
+
+	public boolean processTimedOut() {
+		return hitTimeout;
+	}
+
+	@Override
+	public String toString() {
+		return commandLineToString(getBuilderCommand());
+	}
+
+	private List<String> getBuilderCommand() {
+		return leakPrevention.cleanUpArguments(builder.command());
+	}
+
+	private static String commandLineToString(List<String> commandLine) {
+		StringBuilder sb = new StringBuilder();
+		boolean first = true;
+		for (String s : commandLine) {
+			boolean tricky = s.isEmpty() || s.contains(" ") ;
+
+			if (!first)
+				sb.append(" ");
+			first = false;
+			if (tricky)
+				sb.append("\"");
+
+			sb.append(s.replace("\"", "\\\""));
+
+			if (tricky)
+				sb.append("\"");
+		}
+		return sb.toString();
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/process/Builder.java

@@ -0,0 +1,81 @@
+package com.semmle.util.process;
+
+import java.io.File;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import com.semmle.util.io.StreamMuncher;
+import com.semmle.util.logging.Streams;
+
+public class Builder extends AbstractProcessBuilder {
+
+	private final OutputStream err;
+	private final OutputStream out;
+	protected StreamMuncher errMuncher;
+	protected StreamMuncher outMuncher;
+
+	public Builder(OutputStream out, OutputStream err, File cwd, String... args) {
+		this(out, err, cwd, null, args);
+	}
+
+	public Builder(OutputStream out, OutputStream err, File cwd,
+			Map<String, String> env, String... args) {
+		this(Arrays.asList(args), out, err, env, cwd);
+	}
+
+	public Builder(List<String> args, OutputStream out, OutputStream err) {
+		this(args, out, err, null, null);
+	}
+
+	public Builder(List<String> args, OutputStream out, OutputStream err,
+			File cwd) {
+		this(args, out, err, null, cwd);
+	}
+
+	public Builder(List<String> args, OutputStream out, OutputStream err,
+			Map<String, String> env) {
+		this(args, out, err, env, null);
+	}
+
+	public Builder(List<String> args, OutputStream out, OutputStream err,
+			Map<String, String> env, File cwd) {
+		super(args, cwd, env);
+		this.out = out;
+		this.err = err;
+	}
+
+	/**
+	 * Convenience method that executes the given command line in the current
+	 * working directory with the current environment, blocking until
+	 * completion. The process's output stream is redirected to System.out, and
+	 * its error stream to System.err. It returns the exit code of the command.
+	 */
+	public static int run(List<String> commandLine) {
+		return new Builder(commandLine, Streams.out(), Streams.err()).execute();
+	}
+
+	@Override
+	protected void cleanupOutputHandling() {
+		// wait for munchers to finish munching.
+		long deadline = 1000*MUNCH_TIMEOUT;
+		// note: check that munchers are not null, in case setupOutputHandling was
+		// not called to initialize them
+		if(outMuncher != null) {
+			waitForMuncher("output", outMuncher,deadline);
+		}
+		if(errMuncher != null) {
+			waitForMuncher("error", errMuncher,deadline);
+		}
+	}
+
+	@Override
+	protected void setupOutputHandling(InputStream stdout, InputStream stderr) {
+		errMuncher = new StreamMuncher(stderr, err);
+		errMuncher.start();
+		outMuncher = new StreamMuncher(stdout, out);
+		outMuncher.start();
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/process/Env.java

@@ -0,0 +1,699 @@
+package com.semmle.util.process;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Stack;
+import java.util.TreeMap;
+
+import com.semmle.util.exception.Exceptions;
+import com.semmle.util.expansion.ExpansionEnvironment;
+
+/**
+ * Helper methods for finding out environment properties like the OS type.
+ */
+public class Env {
+	/**
+	 * Enum for commonly used environment variables.
+	 *
+	 * <p>
+	 * The intention is that the name of the enum constant is the same as the environment
+	 * variable itself. This means that the <code>toString</code> method does the right thing,
+	 * as does calling {@link Enum#name() }.
+	 * </p>
+	 *
+	 * <p>
+	 * Should you wish to rename an environment variable (which you're unlikely to, due to the
+	 * fact that there are many non-Java consumers), you can do a rename refactoring to make the
+	 * Java consumers do the right thing.
+	 * </p>
+	 */
+	public enum Var {
+		/*
+		 * Core toolchain variables
+		 */
+		/**
+		 * The location of the user's configuration files, including project configurations,
+		 * dashboard configurations, team insight configurations, licenses etc.
+		 */
+		SEMMLE_HOME,
+		/**
+		 * The location of the user's data, including snapshots, built dashboards, team
+		 * insight data, etc.
+		 */
+		SEMMLE_DATA,
+		/**
+		 * The location of any caches used by the toolchain, including compilation caches, trap caches, etc.
+		 */
+		SEMMLE_CACHE,
+		/**
+		 * If running from a git tree, the root of the tree.
+		 */
+		SEMMLE_GIT_ROOT,
+		/**
+		 * The root from which relative paths in a DOOD file are resolved.
+		 */
+		SEMMLE_QUERY_ROOT,
+		/**
+		 * The directory where lock files are kept.
+		 */
+		SEMMLE_LOCK_DIR,
+		/**
+		 * The directory which will be checked for licenses.
+		 */
+		SEMMLE_LICENSE_DIR,
+		/**
+		 * The location where our queries are kept.
+		 */
+		ODASA_QUERIES,
+		/**
+		 * Whether we are running in 'prototyping mode'.
+		 */
+		ODASA_PROTOTYPE_MODE,
+		/**
+		 * The location of the default compilation cache, as a space-separated list of URIs.
+		 *
+		 * Multiple entries are tried in sequence.
+		 */
+		SEMMLE_COMPILATION_CACHE,
+		/**
+		 * Override the versions used in compilation caching.
+		 * 
+		 * This is useful for testing without modifying the version manually.
+		 */
+		SEMMLE_OVERRIDE_OPTIMISER_VERSION,
+		/**
+		 * If set, do not use compilation caching.
+		 */
+		SEMMLE_NO_COMPILATION_CACHING,
+		/**
+		 * If set, use this as the size of compilation caches, in bytes. If set to 'INFINITY', no
+		 * limit will be placed on the size.
+		 */
+		SEMMLE_COMPILATION_CACHE_SIZE,
+
+		/*
+		 * Other toolchain variables
+		 */
+		SEMMLE_JAVA_HOME,
+		ODASA_JAVA_HOME,
+		ODASA_TRACER_CONFIGURATION,
+		/**
+		 * The Java tracer agent to propagate to JVM processes.
+		 */
+		SEMMLE_JAVA_TOOL_OPTIONS,
+		/**
+		 * Whether to run jar-based subprocesses in-process instead.
+		 */
+		ODASA_IN_PROCESS,
+		/**
+		 * The executable to use for importing trap files.
+		 */
+		SEMMLE_TRAP_IMPORTER,
+		SEMMLE_PRESERVE_SYMLINKS,
+		SEMMLE_PATH_TRANSFORMER,
+
+		/*
+		 * Environment variables for password for credential stores.
+		 * Either is accepted to allow a single entry point in the code
+		 * while documenting as appropriate for the audience.
+		 */
+		SEMMLE_CREDENTIALS_PASSWORD,
+		LGTM_CREDENTIALS_PASSWORD,
+
+		/*
+		 *
+		 * Internal config variables
+		 */
+		/**
+		 * Extra arguments to pass to JVMs launched by Semmle tools.
+		 */
+		SEMMLE_JAVA_ARGS,
+		/**
+		 * A list of log levels to set, of the form:
+		 * "foo.bar=TRACE,bar.baz=DEBUG"
+		 */
+		SEMMLE_LOG_LEVELS,
+		/**
+		 * The default heap size for commands that accept a ram parameter.
+		 */
+		SEMMLE_DEFAULT_HEAP_SIZE,
+		SEMMLE_MAX_RAM_MB,
+		/**
+		 * Whether to disable asynchronous logging in the query server (otherwise it may drop messages).
+		 */
+		SEMMLE_SYNCHRONOUS_LOGGING,
+		/**
+		 * Whether or not to use memory mapping
+		 */
+		SEMMLE_MEMORY_MAPPING,
+		SEMMLE_METRICS_DIR,
+		/**
+		 * Whether we are running in our own unit tests.
+		 */
+		SEMMLE_UNIT_TEST_MODE,
+		/**
+		 * Whether to include the source QL in a QLO.
+		 */
+		SEMMLE_DEBUG_QL_IN_QLO,
+		/**
+		 * Whether to enable extra assertions
+		 */
+		ODASA_ASSERTIONS,
+		/**
+		 * A file containing extra variables for ExpansionEnvironments.
+		 */
+		ODASA_EXTRA_VARIABLES,
+		ODASA_TUNE_GC,
+		/**
+		 * Whether to run PI in hosted mode.
+		 */
+		SEMMLE_ODASA_DEBUG,
+		/**
+		 * The python executable to use for Qltest.
+		 */
+		SEMMLE_PYTHON,
+		/**
+		 * The platform we are running on; one of "linux", "osx" and "win".
+		 */
+		SEMMLE_PLATFORM,
+		/**
+		 * PATH to use to look up tooling required by macOS Relocator scripts.
+		 */
+		CODEQL_TOOL_PATH,
+		/**
+		 * This can override the heuristics for BDD factory resetting. Most useful for measurements
+		 * and debugging.
+		 */
+		CODEQL_BDD_RESET_FRACTION,
+
+		/**
+		 * How many TRAPLinker errors to report.
+		 */
+		SEMMLE_MAX_TRAP_ERRORS,
+
+		/**
+		 * How many tuples to accumulate in memory before pushing to disk.
+		 */
+		SEMMLE_MAX_TRAP_INMEMORY_TUPLES,
+		/**
+		 * How many files to merge at each merge step.
+		 */
+		SEMMLE_MAX_TRAP_MERGE,
+
+		/*
+		 * Variables used by extractors.
+		 */
+		/**
+		 * Whether the C++ extractor should copy executables before
+		 * running them (works around System Integrity Protection
+		 * on OS X 10.11+).
+		 */
+		SEMMLE_COPY_EXECUTABLES,
+		/**
+		 * When SEMMLE_COPY_EXECUTABLES is in operation, where to
+		 * create the directory to copy the executables to.
+		 */
+		SEMMLE_COPY_EXECUTABLES_SUPER_ROOT,
+		/**
+		 * When SEMMLE_COPY_EXECUTABLES is in operation, the
+		 * directory we are copying executables to.
+		 */
+		SEMMLE_COPY_EXECUTABLES_ROOT,
+		/**
+		 * The executable which should be used as an implicit runner on Windows.
+		 */
+		SEMMLE_WINDOWS_RUNNER_BINARY,
+		/**
+		 * Verbosity level for the Java interceptor.
+		 */
+		SEMMLE_INTERCEPT_VERBOSITY,
+		/**
+		 * Verbosity level for the Java extractor.
+		 */
+		ODASA_JAVAC_VERBOSE,
+		/**
+		 * Whether to use class origin tracking for the Java extractor.
+		 */
+		ODASA_JAVA_CLASS_ORIGIN_TRACKING,
+		ODASA_JAVAC_CORRECT_EXCEPTIONS,
+		ODASA_JAVAC_EXTRA_CLASSPATH,
+		ODASA_NO_ECLIPSE_BUILD,
+
+		/*
+		 * Variables set during snapshot builds
+		 */
+		/**
+		 * The location of the project being built.
+		 */
+		ODASA_PROJECT,
+		/**
+		 * The location of the snapshot being built.
+		 */
+		ODASA_SRC,
+		ODASA_DB,
+		ODASA_OUTPUT,
+		ODASA_SUBPROJECT_THREADS,
+
+		/*
+		 * Layout variables
+		 */
+		ODASA_CPP_LAYOUT,
+		ODASA_CSHARP_LAYOUT,
+		ODASA_PYTHON_LAYOUT,
+		ODASA_JAVASCRIPT_LAYOUT,
+
+		/*
+		 * External variables
+		 */
+		JAVA_HOME,
+		PATH,
+		LINUX_VARIANT,
+
+		/*
+		 * If set, use this proxy for HTTP requests
+		 */
+		HTTP_PROXY,
+		http_proxy,
+
+		/*
+		 * If set, use this proxy for HTTPS requests
+		 */
+		HTTPS_PROXY,
+		https_proxy,
+
+		/*
+		 * If set, ignore the variables above and do not use any proxies for requests
+		 */
+		NO_PROXY,
+		no_proxy,
+
+		/*
+		 * Variables set by the codeql-action. All variables will
+		 * be unset if the CLI is not in the context of the
+		 * codeql-action.
+		 */
+
+		/**
+		 * Either {@code actions} or {@code runner}.
+		 */
+		CODEQL_ACTION_RUN_MODE,
+
+		/**
+		 * Semantic version of the codeql-action.
+		 */
+		CODEQL_ACTION_VERSION,
+		/*
+		 * tracer variables
+		 */
+		/**
+		 * Colon-separated list of enabled tracing languages
+		 */
+		CODEQL_TRACER_LANGUAGES,
+		/**
+		 * Path to the build-tracer log file
+		 */
+		CODEQL_TRACER_LOG,
+		/**
+		 * Prefix to a language-specific root directory
+		 */
+		CODEQL_TRACER_ROOT_,
+
+		;
+	}
+
+	private static final int DEFAULT_RAM_MB_32 = 1024;
+	private static final int DEFAULT_RAM_MB = 4096;
+	private static final Env instance = new Env();
+
+	private final Stack<Map<String, String>> envVarContexts;
+
+	public static synchronized Env systemEnv() {
+		return instance;
+	}
+
+	/**
+	 * Create an instance of Env containing no variables. Intended for use in
+	 * testing to isolate the test from the local machine environment.
+	 */
+	public static Env emptyEnv() {
+		Env env = new Env();
+		env.envVarContexts.clear();
+		env.envVarContexts.push(Collections.unmodifiableMap(makeContext()));
+		return env;
+	}
+
+	private static Map<String, String> makeContext() {
+		if (getOS().equals(OS.WINDOWS)) {
+			// We want to compare in the same way Windows does, which means
+			// upper-casing. For example, '_' needs to come after 'Z', but
+			// would come before 'z'.
+			return new TreeMap<>((a, b) -> a.toUpperCase(Locale.ENGLISH).compareTo(b.toUpperCase(Locale.ENGLISH)));
+		} else {
+			return new LinkedHashMap<>();
+		}
+	}
+
+	public Env() {
+		envVarContexts = new Stack<>();
+		Map<String, String> env = makeContext();
+		try {
+			env.putAll(System.getenv());
+		} catch (SecurityException ex) {
+			Exceptions.ignore(ex, "Treat an inaccessible environment variable as not existing");
+		}
+		envVarContexts.push(Collections.unmodifiableMap(env));
+	}
+
+	public synchronized void unsetAll(Collection<String> names) {
+		if (!names.isEmpty()) {
+			Map<String, String> map = envVarContexts.pop();
+			map = new LinkedHashMap<>(map);
+			for (String name : names)
+				map.remove(name);
+			envVarContexts.push(Collections.unmodifiableMap(map));
+		}
+	}
+
+	public synchronized Map<String, String> getenv() {
+		return envVarContexts.peek();
+	}
+
+	/**
+	 * Get the value of an environment variable, or <code>null</code> if
+	 * the environment variable is not set. WARNING: not all systems may
+	 * make a difference between an empty variable or <code>null</code>,
+	 * so don't rely on that behavior.
+	 */
+	public synchronized String get(Var var) {
+		return get(var.name());
+	}
+
+	/**
+	 * Get the value of an environment variable, or <code>null</code> if
+	 * the environment variable is not set. WARNING: not all systems may
+	 * make a difference between an empty variable or <code>null</code>,
+	 * so don't rely on that behavior.
+	 */
+	public synchronized String get(String envVarName) {
+		return getenv().get(envVarName);
+	}
+
+	/**
+	 * Get the non-empty value of an environment variable, or <code>null</code>
+	 * if the environment variable is not set or set to an empty value.
+	 */
+	public synchronized String getNonEmpty(Var var) {
+		return getNonEmpty(var.name());
+	}
+
+	/**
+	 * Get the value of an environment variable, or the empty string if it is not
+	 * set.
+	 */
+	public synchronized String getPossiblyEmpty(String envVarName) {
+		String got = getenv().get(envVarName);
+		return got != null ? got : "";
+	}
+
+	/**
+	 * Get the non-empty value of an environment variable, or <code>null</code>
+	 * if the environment variable is not set or set to an empty value.
+	 */
+	public synchronized String getNonEmpty(String envVarName) {
+		String s = get(envVarName);
+		return s == null || s.isEmpty() ? null : s;
+	}
+
+	/**
+	 * Gets the value of the first environment variable among <code>envVarNames</code>
+	 * whose value is non-empty, or <code>null</code> if all variables have empty values.
+	 */
+	public synchronized String getFirstNonEmpty(String... envVarNames) {
+		for (String envVarName : envVarNames) {
+			String s = getNonEmpty(envVarName);
+			if (s != null)
+				return s;
+		}
+		return null;
+	}
+
+	/**
+	 * Gets the value of the first environment variable among <code>envVars</code>
+	 * whose value is non-empty, or <code>null</code> if all variables have empty values.
+	 */
+	public synchronized String getFirstNonEmpty(Var... envVars) {
+		String[] envVarNames = new String[envVars.length];
+		for (int i = 0; i < envVars.length; ++i)
+			envVarNames[i] = envVars[i].name();
+		return getFirstNonEmpty(envVarNames);
+	}
+
+	/**
+	 * Read a boolean from the given environment variable. If the variable
+	 * is not set, then return <code>false</code>. Otherwise, interpret the
+	 * environment variable using {@link Boolean#parseBoolean(String)}.
+	 */
+	public boolean getBoolean(Var var) {
+		return getBoolean(var.name());
+	}
+
+	/**
+	 * Read a boolean from the given environment variable name. If the variable
+	 * is not set, then return <code>false</code>. Otherwise, interpret the
+	 * environment variable using {@link Boolean#parseBoolean(String)}.
+	 */
+	public boolean getBoolean(String envVarName) {
+		return getBoolean(envVarName, false);
+	}
+
+	/**
+	 * Read a boolean from the given environment variable. If the variable
+	 * is not set, then return <code>def</code>. Otherwise, interpret the
+	 * environment variable using {@link Boolean#parseBoolean(String)}.
+	 */
+	public boolean getBoolean(Var var, boolean def) {
+		return getBoolean(var.name(), def);
+	}
+
+	/**
+	 * Read a boolean from the given environment variable name. If the variable
+	 * is not set, then return <code>def</code>. Otherwise, interpret the
+	 * environment variable using {@link Boolean#parseBoolean(String)}.
+	 */
+	public boolean getBoolean(String envVarName, boolean def) {
+		String v = get(envVarName);
+		return v == null ? def : Boolean.parseBoolean(v);
+	}
+
+	/**
+	 * Read an integer setting from the given environment variable name. If the
+	 * variable is not set, or fails to parse, return the supplied default value.
+	 */
+	public int getInt(Var var, int defaultValue) {
+		return getInt(var.name(), defaultValue);
+	}
+
+	/**
+	 * Read an integer setting from the given environment variable name. If the
+	 * variable is not set, or fails to parse, return the supplied default value.
+	 */
+	public int getInt(String envVarName, int defaultValue) {
+		String value = get(envVarName);
+		if (value == null)
+			return defaultValue;
+
+		try {
+			return Integer.parseInt(value);
+		} catch (NumberFormatException e) {
+			Exceptions.ignore(e, "We'll just use the default value.");
+			return defaultValue;
+		}
+	}
+
+	/**
+	 * Enter a new context for environment variables, with the given
+	 * new variable values. The values will override the current environment
+	 * values if they define the same variables.
+	 */
+	public synchronized void pushEnvironmentContext(Map<String, String> addedValues) {
+		Map<String, String> newValues = makeContext();
+		newValues.putAll(envVarContexts.peek());
+		newValues.putAll(addedValues);
+		envVarContexts.push(Collections.unmodifiableMap(newValues));
+	}
+
+	/**
+	 * Leave a context for environment variables that was created with
+	 * <code>pushEnvironmentContext</code>
+	 */
+	public synchronized void popEnvironmentContext() {
+		envVarContexts.pop();
+	}
+
+	/**
+	 * Add all the custom environment variables to a process builder, so that
+	 * they are passed on to the child process.
+	 */
+	public synchronized void addEnvironmentToNewProcess(ProcessBuilder builder) {
+		if (envVarContexts.size() > 1)
+			builder.environment().putAll(envVarContexts.peek());
+	}
+
+	public synchronized void addEnvironmentToNewEnv(ExpansionEnvironment env) {
+		if (envVarContexts.size() > 1)
+			env.defineVars(envVarContexts.peek());
+	}
+
+	/**
+	 * Get a string representing the OS type. This
+	 * is not guaranteed to have any particular form, and
+	 * is for displaying to users. Might return <code>null</code> if
+	 * the property is not defined by the JVM.
+	 */
+	public static String getOSName() {
+		return System.getProperty("os.name");
+	}
+
+	/**
+	 * Determine which OS is currently being run (somewhat best-effort).
+	 * Does not determine whether a program is being run under Cygwin
+	 * or not - Windows will be the OS even under Cygwin.
+	 */
+	public static OS getOS() {
+		String name = getOSName();
+		if (name == null)
+			return OS.UNKNOWN;
+		if (name.contains("Windows"))
+			return OS.WINDOWS;
+		else if (name.contains("Mac OS X"))
+			return OS.MACOS;
+		else if (name.contains("Linux"))
+			return OS.LINUX;
+		else
+			// Guess that we are probably some Unix flavour
+			return OS.UNKNOWN_UNIX;
+	}
+
+	/**
+	 * Kinds of operating systems. A notable absence is Cygwin: this just
+	 * gets reported as Windows.
+	 */
+	public static enum OS {
+		WINDOWS(false, false), LINUX(true, true), MACOS(false, true), UNKNOWN_UNIX(true, true), UNKNOWN(true, true),;
+
+		private final boolean fileSystemCaseSensitive;
+		private final boolean envVarsCaseSensitive;
+
+		private OS(boolean fileSystemCaseSensitive, boolean envVarsCaseSensitive) {
+			this.fileSystemCaseSensitive = fileSystemCaseSensitive;
+			this.envVarsCaseSensitive = envVarsCaseSensitive;
+		}
+
+		/**
+		 * Get an OS value from the short display name. Acceptable
+		 * inputs (case insensitive) are: Windows, Linux, MacOS or
+		 * Mac OS.
+		 *
+		 * @throws IllegalArgumentException if the given name does not
+		 *         correspond to an OS
+		 */
+		public static OS fromDisplayName(String name) {
+			if (name != null) {
+				name = name.toUpperCase();
+				if ("WINDOWS".equals(name))
+					return WINDOWS;
+				if ("LINUX".equals(name))
+					return LINUX;
+				if ("MACOS".equals(name.replace(" ", "")))
+					return MACOS;
+			}
+			throw new IllegalArgumentException("No OS type found with name " + name);
+		}
+
+		public boolean isFileSystemCaseSensitive() {
+			return fileSystemCaseSensitive;
+		}
+
+		public boolean isEnvironmentCaseSensitive() {
+			return envVarsCaseSensitive;
+		}
+
+		/** The short name of this operating system, in the style of {@link Var#SEMMLE_PLATFORM}. */
+		public String getShortName() {
+			switch (this) {
+			case WINDOWS:
+				return "win";
+			case LINUX:
+				return "linux";
+			case MACOS:
+				return "osx";
+			default:
+				return "unknown";
+			}
+		}
+	}
+
+	public static enum Architecture {
+		X86(true, false), X64(false, true), UNDETERMINED(false, false);
+
+		private final boolean is32Bit;
+		private final boolean is64Bit;
+
+		private Architecture(boolean is32Bit, boolean is64Bit) {
+			this.is32Bit = is32Bit;
+			this.is64Bit = is64Bit;
+		}
+
+		/** Is this definitely a 32-bit architecture? */
+		public boolean is32Bit() {
+			return is32Bit;
+		}
+
+		/** Is this definitely a 64-bit architecture? */
+		public boolean is64Bit() {
+			return is64Bit;
+		}
+	}
+
+	/**
+	 * Try to detect whether the JVM is 32-bit or 64-bit. Since there is no documented,
+	 * portable way to do this it is best effort.
+	 */
+	public Architecture tryDetermineJvmArchitecture() {
+		String value = System.getProperty("sun.arch.data.model");
+		if ("32".equals(value))
+			return Architecture.X86;
+		else if ("64".equals(value))
+			return Architecture.X64;
+
+		// Look at the max heap value - if >= 4G we *must* be in 64-bit
+		long maxHeap = Runtime.getRuntime().maxMemory();
+		if (maxHeap < Long.MAX_VALUE && maxHeap >= 4096L << 20)
+			return Architecture.X64;
+
+		// Try to get the OS arch - it *appears* to give JVM bitness
+		String osArch = System.getProperty("os.arch");
+		if ("x86".equals(osArch) || "i386".equals(osArch))
+			return Architecture.X86;
+		else if ("x86_64".equals(osArch) || "amd64".equals(osArch))
+			return Architecture.X64;
+
+		return Architecture.UNDETERMINED;
+	}
+
+	/**
+	 * Get the default amount of ram to use for new JVMs, depending on the
+	 * current architecture. If it looks like we're running on a 32-bit
+	 * machine, the result is sufficiently small to be representable.
+	 */
+	public int defaultRamMb() {
+		return getInt(
+				Var.SEMMLE_DEFAULT_HEAP_SIZE,
+				tryDetermineJvmArchitecture().is32Bit() ? DEFAULT_RAM_MB_32 : DEFAULT_RAM_MB);
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/process/LeakPrevention.java

@@ -0,0 +1,95 @@
+package com.semmle.util.process;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.List;
+
+public abstract class LeakPrevention {
+
+	public abstract List<String> cleanUpArguments(List<String> args);
+
+	/**
+	 * What to put in place of any suppressed arguments.
+	 */
+	static final String REPLACEMENT_STRING = "*****";
+
+	/**
+	 * Hides all arguments. Will only show the command name.
+	 * e.g. "foo bar baz" is changed to "foo"
+	 */
+	public static final LeakPrevention ALL = new LeakPrevention() {
+		@Override
+		public List<String> cleanUpArguments(List<String> args) {
+			return args.isEmpty() ? args : Collections.singletonList(args.get(0));
+		}
+	};
+
+	/**
+	 * Does not hide any arguments.
+	 */
+	public static final LeakPrevention NONE = new LeakPrevention() {
+		@Override
+		public List<String> cleanUpArguments(List<String> args) {
+			return args;
+		}
+	};
+
+	/**
+	 * Hides the arguments at the given indexes.
+	 */
+	public static LeakPrevention suppressedArguments(int... args) {
+		if (args.length == 0)
+			return NONE;
+
+		final BitSet suppressed = new BitSet();
+		for (int index : args) {
+			suppressed.set(index);
+		}
+
+		return new LeakPrevention() {
+			@Override
+			public List<String> cleanUpArguments(List<String> args) {
+				List<String> result = new ArrayList<>(args.size());
+				int index = 0;
+				for (String arg : args) {
+					if (suppressed.get(index))
+						result.add(REPLACEMENT_STRING);
+					else
+						result.add(arg);
+					index++;
+				}
+				return result;
+			}
+		};
+	}
+
+	/**
+	 * Hides the given string from any arguments that it appears in.
+	 * The substring will be replaced while leaving the rest of the
+	 * argument unmodified.
+	 * <p>
+	 * There are some potential pitfalls to be aware of when using this
+	 * method.
+	 * <ul>
+	 * <li>This only suppresses exact textual matches. If the argument that
+	 * appears is only derived from the secret instead of being an exact
+	 * copy then it will not be suppressed.
+	 * <li>If the secret value appears elsewhere in a known string, then it
+	 * could leak the contents of the secret because the viewer knows what
+	 * should have been there in the known case.
+	 * </ul>
+	 */
+	public static LeakPrevention suppressSubstring(final String substringToSuppress) {
+		return new LeakPrevention() {
+			@Override
+			public List<String> cleanUpArguments(List<String> args) {
+				List<String> result = new ArrayList<>(args.size());
+				for (String arg : args) {
+					result.add(arg.replace(substringToSuppress, REPLACEMENT_STRING));
+				}
+				return result;
+			}
+		};
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/projectstructure/ProjectLayout.java

@@ -0,0 +1,529 @@
+package com.semmle.util.projectstructure;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.semmle.util.data.StringUtil;
+import com.semmle.util.exception.CatastrophicError;
+import com.semmle.util.exception.UserError;
+import com.semmle.util.io.WholeIO;
+
+/**
+ * A project-layout file optionally begins with an '@'
+ * followed by the name the project should be renamed to.
+ * Optionally, it can then be followed by a list of
+ * include/exclude patterns (see below) which are kept
+ * as untransformed paths. This is followed by one or
+ * more clauses. Each clause has the following form:
+ *
+ * #virtual-path
+ * path/to/include
+ * another/path/to/include
+ * -/path/to/include/except/this
+ *
+ * i.e. one or more paths (to include) and zero or more paths
+ * prefixed by minus-signs (to exclude).
+ */
+public class ProjectLayout
+{
+	public static final char PROJECT_NAME_PREFIX = '@';
+
+	private String project;
+
+	/**
+	 * Map from virtual path prefixes (following the '#' in the project-layout)
+	 * to the sequence of patterns that fall into that section. Declared as a
+	 * {@link LinkedHashMap} since iteration order matters -- we process blocks in
+	 * the same order as they occur in the project-layout.
+	 */
+	private final LinkedHashMap<String, Section> sections = new LinkedHashMap<String, Section>();
+
+	/**
+	 * A file name, or similar string, to use in error messages so that the
+	 * user knows what to fix.
+	 */
+	private String source;
+
+	/**
+	 * Load a project-layout file.
+	 *
+	 * @param file the project-layout to load
+	 */
+	public ProjectLayout(File file) {
+		this(StringUtil.lines(new WholeIO().strictread(file)), file.toString());
+	}
+
+	/**
+	 * Construct a project-layout object from an array of strings, each
+	 * corresponding to one line of the project-layout. This constructor
+	 * is for testing. For other uses see {@link ProjectLayout#ProjectLayout(File)}.
+	 *
+	 * @param lines the lines of the project-layout
+	 */
+	public ProjectLayout(String... lines) {
+		this(lines, null);
+	}
+
+	private ProjectLayout(String[] lines, String source) {
+		this.source = source;
+		String virtual = "";
+		Section section = new Section("");
+		sections.put("", section);
+		int num = 0;
+		for (String line : lines) {
+			num++;
+			line = line.trim();
+			if (line.isEmpty())
+				continue;
+			switch (line.charAt(0)) {
+			case PROJECT_NAME_PREFIX:
+				if (project != null)
+					throw error("Only one project name is allowed", source, num);
+				project = tail(line);
+				break;
+			case '#':
+				virtual = tail(line);
+				if (sections.containsKey(virtual))
+					throw error("Duplicate virtual path prefix " + virtual, source, num);
+				section = new Section(virtual);
+				sections.put(virtual, section);
+				break;
+			case '-':
+				section.add(new Rewrite(tail(line), source, num));
+				break;
+			default:
+				section.add(new Rewrite(line, virtual, source, num));
+			}
+		}
+	}
+
+	private static String tail(String line) {
+		return line.substring(1).trim();
+	}
+
+	/**
+	 * Get the project name, if specified by the project-layout. This
+	 * method should only be called if it is guaranteed that the
+	 * project-layout will contain a project name, and it throws
+	 * a {@link UserError} if it doesn't.
+	 * @return the project name -- guaranteed not <code>null</code>.
+	 * @throws UserError if the project-layout file did not specify a
+	 *     project name.
+	 */
+	public String projectName() {
+		if (project == null)
+			throw error("No project name is defined", source);
+		return project;
+	}
+
+	/**
+	 * Get the project name, if specified by the project-layout file.
+	 * If the file contains no renaming specification, return the
+	 * given default value.
+	 * @param defaultName The name to use if the project-layout doesn't
+	 *    specify a target project name.
+	 * @return the specified name or default value.
+	 */
+	public String projectName(String defaultName) {
+		return project == null ? defaultName : project;
+	}
+
+	/**
+	 * @return the section headings (aka virtual paths)
+	 */
+	public List<String> sections() {
+		List<String> result = new ArrayList<String>();
+		result.addAll(sections.keySet());
+		return result;
+	}
+
+	/**
+	 * Determine whether or not a particular section in this
+	 * project-layout is empty (has no include/exclude patterns).
+	 *
+	 * @param section the name of the section
+	 * @return <code>true</code> if the section is empty
+	 */
+	public boolean sectionIsEmpty(String section) {
+		if (!sections.containsKey(section))
+			throw new CatastrophicError("Section does not exist: " + section);
+		return sections.get(section).isEmpty();
+	}
+
+	/**
+	 * Reaname a section in this project-layout.
+	 *
+	 * @param oldName the old name of the section
+	 * @param newName the new name
+	 */
+	public void renameSection(String oldName, String newName) {
+		if (!sections.containsKey(oldName))
+			throw new CatastrophicError("Section does not exist: " + oldName);
+		Section section = sections.remove(oldName);
+		section.rename(newName);
+		sections.put(newName, section);
+	}
+
+	/**
+	 * Return a project-layout file for just one of the sections in this
+	 * project-layout. This is done by copying all the rules from the
+	 * section, and changing the section heading (beginning with '#')
+	 * to a project name (beginning with '@').
+	 *
+	 * @param sectionName the section to create a project-layout from
+	 * @return the text of the newly created project-layout
+	 */
+	public String subLayout(String sectionName) {
+		Section section = sections.get(sectionName);
+		if (section == null)
+			throw new CatastrophicError("Section does not exist: " + section);
+		return section.toLayout();
+	}
+
+	/**
+	 * Maps a path to its corresponding artificial path according to the
+	 * rules in this project-layout. If the path is excluded (either
+	 * explicitly, or because it is not mentioned in the project-layout)
+	 * then <code>null</code> is returned.
+	 * <p>
+	 * Paths should start with a leading forward-slash
+	 *
+	 * @param path the path to map
+	 * @return the artificial path, or <code>null</code> if the path is excluded
+	 */
+	public String artificialPath(String path) {
+		// If there is no leading slash, the path does not conform to the expected
+		// format and there is no match. (An exception is made for a completely
+		// empty string, which will get the sole prefix '/' and be mapped as usual).
+		if (path.length() > 0 && path.charAt(0) != '/')
+			return null;
+		List<String> prefixes = Section.prefixes(path);
+		for (Section section : sections.values()) {
+			Rewrite rewrite = section.match(prefixes);
+			String rewritten = null;
+			if (rewrite != null)
+				rewritten = rewrite.rewrite(path);
+			if (rewritten != null)
+				return rewritten;
+		}
+		return null;
+	}
+
+	/**
+	 * Checks whether a path should be included in the project specified by
+	 * this file. A file is included if it is mapped to some location.
+	 * <p>
+	 * Paths should start with a leading forward-slash
+	 *
+	 * @param path the path to check
+	 * @return <code>true</code> if the path should be included
+	 */
+	public boolean includeFile(String path) {
+		return artificialPath(path) != null;
+	}
+
+	public void writeTo(Writer writer) throws IOException {
+		if (project != null) {
+			writer.write(PROJECT_NAME_PREFIX);
+			writer.write(project);
+			writer.write("\n");
+		}
+		for (Section section : sections.values()) {
+			if (!section.virtual.isEmpty()) {
+				writer.write("#");
+				writer.write(section.virtual);
+				writer.write("\n");
+			}
+			section.outputRules(writer);
+		}
+	}
+
+	public void addPattern(String section, String pattern) {
+		if (pattern == null || pattern.isEmpty()) {
+			throw new IllegalArgumentException("ProjectLayout.addPattern: pattern must be a non-empty string");
+		}
+		boolean exclude = pattern.charAt(0) == '-';
+		Rewrite rewrite = exclude ?
+			new Rewrite(pattern.substring(1), null, 0) :
+			new Rewrite(pattern, section, null, 0);
+		Section s = sections.get(section);
+		if (s == null) {
+			s = new Section(section);
+			sections.put(section, s);
+		}
+		s.add(rewrite);
+	}
+
+	private static UserError error(String message, String source) {
+		return error(message, source, 0);
+	}
+
+	private static UserError error(String message, String source, int line) {
+		if (source == null)
+			return new UserError(message);
+		StringBuilder sb = new StringBuilder(message);
+		sb.append(" (");
+		if (line > 0)
+			sb.append("line ").append(line).append(" of ");
+		sb.append(source).append(")");
+		return new UserError(sb.toString());
+	}
+
+	/**
+	 * Each section corresponds to a block beginning with  '#some/path'. There
+	 * is also an initial section for any include/exclude patterns before the
+	 * first '#'.
+	 */
+	private static class Section {
+		private String virtual;
+		private final Map<String, Rewrite> simpleRewrites;
+		private final List<Rewrite> complexRewrites;
+
+		public Section(String virtual) {
+			this.virtual = virtual;
+			simpleRewrites = new LinkedHashMap<String, Rewrite>();
+			complexRewrites = new ArrayList<Rewrite>();
+		}
+
+		public String toLayout() {
+			StringWriter result = new StringWriter();
+			result.append('@').append(virtual).append('\n');
+			try {
+				outputRules(result);
+			} catch (IOException e) {
+				throw new CatastrophicError("StringWriter.append threw an IOException", e);
+			}
+			return result.toString();
+		}
+
+		private void outputRules(Writer writer) throws IOException {
+			List<Rewrite> all = new ArrayList<Rewrite>();
+			all.addAll(simpleRewrites.values());
+			all.addAll(complexRewrites);
+			Collections.sort(all, Rewrite.COMPARATOR);
+			for (Rewrite rewrite : all)
+				writer.append(rewrite.toString()).append('\n');
+		}
+
+		public void rename(String newName) {
+			virtual = newName;
+			for (Rewrite rewrite : simpleRewrites.values())
+				rewrite.virtual = newName;
+			for (Rewrite rewrite : complexRewrites)
+				rewrite.virtual = newName;
+		}
+
+		public void add(Rewrite rewrite) {
+			int index = simpleRewrites.size() + complexRewrites.size();
+			rewrite.setIndex(index);
+			if (rewrite.isSimple())
+				simpleRewrites.put(rewrite.simplePrefix(), rewrite);
+			else
+				complexRewrites.add(rewrite);
+		}
+
+		public boolean isEmpty() {
+			return simpleRewrites.isEmpty() && complexRewrites.isEmpty();
+		}
+
+		private static List<String> prefixes(String path) {
+			List<String> result = new ArrayList<String>();
+			result.add(path);
+			int i = path.length();
+			while (i > 1) {
+				i = path.lastIndexOf('/', i - 1);
+				result.add(path.substring(0, i));
+			}
+			result.add("/");
+			return result;
+		}
+
+		public Rewrite match(List<String> prefixes) {
+			Rewrite best = null;
+			for (String prefix : prefixes) {
+				Rewrite match = simpleRewrites.get(prefix);
+				if (match != null)
+					if (best == null || best.index < match.index)
+						best = match;
+			}
+			// Last matching rewrite 'wins'
+			for (int i = complexRewrites.size() - 1; i >= 0; i--) {
+				Rewrite rewrite = complexRewrites.get(i);
+				if (rewrite.matches(prefixes.get(0))) {
+					if (best == null || best.index < rewrite.index)
+						best = rewrite;
+					// no point continuing
+					break;
+				}
+			}
+			return best;
+		}
+	}
+
+	/**
+	 * Each Rewrite corresponds to a single include or exclude line in the project-layout.
+	 * For example, for following clause there would be three Rewrite objects:
+	 *
+	 * #Source
+	 * /src
+	 * /lib
+	 * -/src/tests
+	 *
+	 * For includes use the two-argument constructor; for excludes the one-argument constructor.
+	 */
+	private static class Rewrite {
+
+		private static final Comparator<Rewrite> COMPARATOR = new Comparator<Rewrite>() {
+
+			@Override
+			public int compare(Rewrite t, Rewrite o) {
+				if (t.index < o.index)
+					return -1;
+				if (t.index == o.index)
+					return 0;
+				return 1;
+			}
+		};
+
+		private int index;
+		private final String original;
+		private final Pattern pattern;
+		private String virtual;
+		private final String simple;
+
+		/**
+		 * The intention is to allow the ** wildcard when followed by a slash only. The
+		 * following should be invalid:
+		 * - a / *** / b (too many stars)
+		 * - a / ** (** at the end should be omitted)
+		 * - a / **b (illegal)
+		 * - a / b** (illegal)
+		 * - ** (the same as a singleton '/')
+		 * This regex matches ** when followed by a non-/ character, or the end of string.
+		 */
+		private static final Pattern verifyStars = Pattern.compile(".*(?:\\*\\*[^/].*|\\*\\*$|[^/]\\*\\*.*)");
+
+		public Rewrite(String exclude, String source, int line) {
+			original = '-' + exclude;
+			if (!exclude.startsWith("/"))
+				exclude = '/' + exclude;
+			if (exclude.indexOf("//") != -1)
+				throw error("Illegal '//' in exclude path", source, line);
+			if (verifyStars.matcher(exclude).matches())
+				throw error("Illegal use of '**' in exclude path", source, line);
+			if (exclude.endsWith("/"))
+				exclude = exclude.substring(0, exclude.length() - 1);
+			pattern = compilePrefix(exclude);
+			exclude = exclude.replace("//", "/");
+			if (exclude.length() > 1 && exclude.endsWith("/"))
+				exclude = exclude.substring(0, exclude.length() - 1);
+			simple = exclude.contains("*") ? null : exclude;
+		}
+
+		public void setIndex(int index) {
+			this.index = index;
+		}
+
+		public Rewrite(String include, String virtual, String source, int line) {
+			original = include;
+			if (!include.startsWith("/"))
+				include = '/' + include;
+			int doubleslash = include.indexOf("//");
+			if (doubleslash != include.lastIndexOf("//"))
+				throw error("More than one '//' in include path", source, line);
+			if (verifyStars.matcher(include).matches())
+				throw error("Illegal use of '**' in include path", source, line);
+			if (!virtual.startsWith("/"))
+				virtual = "/" + virtual;
+			if (virtual.endsWith("/"))
+				virtual = virtual.substring(0, virtual.length() - 1);
+			this.virtual = virtual;
+			this.pattern = compilePrefix(include);
+			include = include.replace("//", "/");
+			if (include.length() > 1 && include.endsWith("/"))
+				include = include.substring(0, include.length() - 1);
+			simple = include.contains("*") ? null : include;
+		}
+
+		/**
+		 * Patterns are matched by translation to regex. The following invariants
+		 * are assumed to hold:
+		 *
+		 *  - The pattern starts with a '/'.
+		 *  - There are no occurrences of '**' that is not surrounded by slashes
+		 *    (unless it is at the start of a pattern).
+		 *  - There is at most one double slash.
+		 *
+		 *  The result of the translation has precisely one capture group, which
+		 *  (after successful matching) will contain the part of the path that
+		 *  should be glued to the virtual prefix.
+		 *
+		 *  It proceeds by starting the capture group either after the double
+		 *  slash or at the start of the pattern, and then replacing '*' with
+		 *  '[^/]*' (meaning any number of non-slash characters) and '/**' with
+		 *  '(?:|/.*)' (meaning empty string or a slash followed by any number of
+		 *  characters including '/').
+		 *
+		 *   The pattern is terminated by the term '(?:/.*|$)', saying 'either the
+		 *   next character is a '/' or the string ends' -- this avoids accidental
+		 *   matching of partial directory/file names.
+		 *
+		 *   <b>IMPORTANT:</b> Run the ProjectLayoutTests when changing this!
+		 */
+		private static Pattern compilePrefix(String pattern) {
+			pattern = StringUtil.escapeStringLiteralForRegexp(pattern, "*");
+			if (pattern.contains("//"))
+				pattern = pattern.replace("//", "(/");
+			else
+				pattern = "(" + pattern;
+			if (pattern.endsWith("/"))
+				pattern = pattern.substring(0, pattern.length() - 1);
+			pattern = pattern.replace("/**", "-///-")
+							.replace("*", "[^/]*")
+							.replace("-///-", "(?:|/.*)");
+			return Pattern.compile(pattern + "(?:/.*|$))");
+		}
+
+		/** Is this rewrite simple? (i.e. contains no wildcards) */
+		public boolean isSimple() {
+			return simple != null;
+		}
+
+		/** Returns the path included/excluded by this rewrite, if it is
+		 * simple, or <code>null</code> if it is not.
+		 *
+		 * @return included/excluded path, or <code>null</code>
+		 */
+		public String simplePrefix() {
+			return simple;
+		}
+
+		public boolean matches(String path) {
+			return pattern.matcher(path).matches();
+		}
+
+		public String rewrite(String path) {
+			if (virtual == null)
+				return null;
+			Matcher matcher = pattern.matcher(path);
+			if (!matcher.matches())
+				return null;
+			return virtual + matcher.group(1);
+		}
+
+		@Override
+		public String toString() {
+			return original;
+		}
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/trap/CompressedFileInputStream.java

@@ -0,0 +1,29 @@
+package com.semmle.util.trap;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import com.semmle.util.zip.MultiMemberGZIPInputStream;
+
+public class CompressedFileInputStream {
+	/**
+	 * Create an input stream for reading the uncompressed data from a (possibly) compressed file, with
+	 * the decompression method chosen based on the file extension.
+	 *
+	 * @param f The compressed file to read
+	 * @return An input stream from which you can read the file's uncompressed data.
+	 * @throws IOException From the underlying decompression input stream.
+	 */
+	public static InputStream fromFile(Path f) throws IOException {
+		InputStream fileInputStream = Files.newInputStream(f);
+		if (f.getFileName().toString().endsWith(".gz")) {
+			return new MultiMemberGZIPInputStream(fileInputStream, 8192);
+		//} else if (f.getFileName().toString().endsWith(".br")) {
+		//	return new BrotliInputStream(fileInputStream);
+		} else {
+			return fileInputStream;
+		}
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TextFile.java

@@ -0,0 +1,125 @@
+package com.semmle.util.trap.dependencies;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.semmle.util.exception.ResourceError;
+import com.semmle.util.io.StreamUtil;
+import com.semmle.util.io.WholeIO;
+import com.semmle.util.trap.CompressedFileInputStream;
+
+public abstract class TextFile {
+	static final String TRAPS = "TRAPS";
+	private static final Pattern HEADER = Pattern.compile("([^\r\n]+?) (\\d\\.\\d)");
+
+	protected String version;
+	protected final Set<String> traps = new LinkedHashSet<String>();
+	
+	protected abstract Set<String> getSet(Path path, String label);
+	protected abstract void parseError(Path path);
+	
+	public TextFile(String version) {
+		this.version = version;
+	}
+	
+	/**
+	 * Load the current text file, checking that it matches the expected header.
+	 *
+	 * <p>
+	 * This method is somewhat performance-sensitive, as at least our C++ extractors
+	 * can generate very large input files. The format is therefore parsed by hand.
+	 * </p>
+	 *
+	 * <p>
+	 * The accepted format consists of:
+	 * <ul>
+	 * <li>Zero or more EOL comments, marked with {@code //}.
+	 * <li>Precisely one header line, of the form {@code $HEADER $VERSION}; this is
+	 * checked against {@code expected_header}.
+	 * <li>Zero or more "file lists", each beginning with the name of a set (see
+	 * {@link #getSet(File, String)}) on a line by itself, followed by file paths,
+	 * one per line.
+	 * </ul>
+	 * 
+	 * <p>
+	 * Empty lines are permitted throughout.
+	 * </p>
+	 */
+	protected void load(String expected_header, Path path) {
+		try (InputStream is = CompressedFileInputStream.fromFile(path);
+				BufferedReader lines = StreamUtil.newUTF8BufferedReader(is)) {
+			boolean commentsPermitted = true;
+			Set<String> currentSet = null;
+			for (String line = lines.readLine(); line != null; line = lines.readLine()) {
+				// Skip empty lines.
+				if (line.isEmpty())
+					continue;
+				// If comments are still permitted, skip comment lines.
+				if (commentsPermitted && line.startsWith("//"))
+					continue;
+				// If comments are still permitted, the first non-comment line is the header.
+				// In addition, we allow no further comments.
+				if (commentsPermitted) {
+					Matcher matcher = HEADER.matcher(line);
+					if (!matcher.matches() || !matcher.group(1).equals(expected_header))
+						parseError(path);
+					commentsPermitted = false;
+					version = matcher.group(2);
+					continue;
+				}
+				// We have a non-blank line; this either names the new set, or is a line that
+				// should be put into the current set.
+				Set<String> newSet = getSet(path, line);
+				if (newSet != null) {
+					currentSet = newSet;
+				} else {
+					if (currentSet == null)
+						parseError(path);
+					else
+						currentSet.add(line);
+				}
+			}
+		} catch (IOException e) {
+			throw new ResourceError("Couldn't read " + path, e);
+		}
+	}
+	
+	/**
+	 * @return the format version of the loaded file
+	 */
+	public String version() {
+		return version;
+	}
+	
+	/**
+	 * Save this object to a file (or throw a ResourceError on failure)
+	 * 
+	 * @param file the file in which to save this object
+	 */
+	public void save(Path file) {
+		new WholeIO().strictwrite(file, toString());
+	}
+	
+	protected void appendHeaderString(StringBuilder sb, String header, String version) {
+		sb.append(header).append(' ').append(version).append('\n');
+	}
+
+	protected void appendSet(StringBuilder sb, String title, Set<String> set) {
+		sb.append('\n').append(title).append('\n');
+		for (String s : set)
+			sb.append(s).append('\n');
+	}
+	
+	protected void appendSingleton(StringBuilder sb, String title, String s) {
+		sb.append('\n').append(title).append('\n');
+		sb.append(s).append('\n');
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TrapDependencies.java

@@ -0,0 +1,109 @@
+package com.semmle.util.trap.dependencies;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.util.AbstractSet;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Set;
+
+import com.semmle.util.exception.ResourceError;
+
+/**
+ * The immediate dependencies of a particular TRAP file
+ */
+public class TrapDependencies extends TextFile
+{
+	static final String TRAP = "TRAP";
+
+	private String trap;
+
+	/**
+	 * Create an empty dependencies node for a TRAP file
+	 */
+	public TrapDependencies(String trap) {
+		super(TrapSet.LATEST_VERSION);
+		this.trap = trap;
+	}
+
+	/**
+	 * Load a TRAP dependencies (.dep) file
+	 * 
+	 * @param file the file to load
+	 */
+	public TrapDependencies(Path file) {
+		super(null);
+		load(TrapSet.HEADER, file);
+		if(trap == null)
+			parseError(file);
+	}
+	
+	@Override
+	protected Set<String> getSet(final Path file, String label) {
+		if(label.equals(TRAP)) {
+			return new AbstractSet<String>() {
+				@Override
+				public Iterator<String> iterator() {
+					return null;
+				}
+				@Override
+				public int size() {
+					return 0;
+				}
+				@Override
+				public boolean add(String s) {
+					if(trap != null)
+						parseError(file);
+					trap = s;
+					return true;
+				}
+			};
+		}
+		if(label.equals(TRAPS)) return traps;
+		return null;
+	}
+	
+	@Override
+	protected void parseError(Path file) {
+		throw new ResourceError("Corrupt TRAP dependencies: " + file);
+	}
+
+	/**
+	 * @return the path of the TRAP with the dependencies stored in this object
+	 *         (relative to the source location)
+	 */
+	public String trapFile() {
+		return trap;
+	}
+
+	/**
+	 * @return the paths of the TRAP file dependencies
+	 *         (relative to the trap directory)
+	 *         
+	 */
+	public Set<String> dependencies() {
+		return Collections.unmodifiableSet(traps);
+	}
+
+	/**
+	 * Add a path to a TRAP file (relative to the trap directory).
+	 * 
+	 * @param trap the path to the trap file to add
+	 */
+	public void addDependency(String trap) {
+		traps.add(trap);
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see java.lang.Object#toString()
+	 */
+	@Override
+	public String toString() {
+		StringBuilder sb = new StringBuilder();
+		appendHeaderString(sb, TrapSet.HEADER, TrapSet.LATEST_VERSION);
+		appendSingleton(sb, TRAP, trap);
+		appendSet(sb, TRAPS, traps);
+		return sb.toString();
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TrapSet.java

@@ -0,0 +1,196 @@
+package com.semmle.util.trap.dependencies;
+
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+import com.semmle.util.exception.ResourceError;
+
+/**
+ * A set of source files and the TRAP files that were generated when
+ * compiling them.
+ * <p>
+ * The set of TRAP files is not necessarily sufficient to create a
+ * consistent database, unless combined with inter-TRAP dependency
+ * information from .dep files (see {@link TrapDependencies}).
+ */
+public class TrapSet extends TextFile
+{
+	static final String HEADER = "TRAP dependencies";
+	static final String LATEST_VERSION = "1.2";
+	static final String SOURCES = "SOURCES";
+	static final String INCLUDES = "INCLUDES";
+	static final String OBJECTS = "OBJECTS";
+	static final String INPUT_OBJECTS = "INPUT_OBJECTS";
+
+	// state
+	private final Set<String> sources = new LinkedHashSet<String>();
+	private final Set<String> includes = new LinkedHashSet<String>();
+	private final Set<String> objects = new LinkedHashSet<String>();
+	private final Set<String> inputObjects = new LinkedHashSet<String>();
+
+	private Path file;
+
+	/**
+	 * Create an empty TRAP set
+	 */
+	public TrapSet() {
+		super(LATEST_VERSION);
+	}
+	
+	@Override
+	protected Set<String> getSet(Path file, String label) {
+		if (label.equals(SOURCES)) return sources;
+		if (label.equals(INCLUDES)) return includes;
+		if (label.equals(OBJECTS)) return objects;
+		if (label.equals(INPUT_OBJECTS)) return inputObjects;
+		if (label.equals(TRAPS)) return traps;
+		return null;
+	}
+
+	/**
+	 * Load a TRAP set (.set) file
+	 * 
+	 * @param path the file to load
+	 */
+	public TrapSet(Path path) {
+		super(null);
+		load(HEADER, path);
+		this.file = path;
+	}
+
+	/**
+	 * Return the most recent file used when loading or saving this
+	 * trap set. If this set was constructed, rather than loaded, and
+	 * has not been saved then the result is <code>null</code>.
+	 *
+	 * @return the file or <code>null</code>
+	 */
+	public Path getFile() {
+		return file;
+	}
+	
+	@Override
+	protected void parseError(Path file) {
+		throw new ResourceError("Corrupt TRAP set: " + file);
+	}
+
+	/**
+	 * @return the paths of the source files contained in this TRAP set
+	 */
+	public Set<String> sourceFiles() {
+		return Collections.unmodifiableSet(sources);
+	}
+
+	/**
+	 * @return the paths to the include files contained in this TRAP set
+	 */
+	public Set<String> includeFiles() {
+		return Collections.unmodifiableSet(includes);
+	}
+
+	/**
+	 * @return the paths of the TRAP files contained in this TRAP set
+	 *         (relative to the trap directory)
+	 *         
+	 */
+	public Set<String> trapFiles() {
+		return Collections.unmodifiableSet(traps);
+	}
+	
+	/**
+	 * @return the object names in this TRAP set
+	 *
+	 */
+	public Set<String> objectNames() {
+		return Collections.unmodifiableSet(objects);
+	}
+
+	/**
+	 * @return the object names in this TRAP set
+	 *
+	 */
+	public Set<String> inputObjectNames() {
+		return Collections.unmodifiableSet(inputObjects);
+	}
+
+	/**
+	 * Add a fully-qualified path to a source-file.
+	 * 
+	 * @param source the path to the source file to add
+	 */
+	public void addSource(String source) {
+		sources.add(source);
+	}
+
+	/**
+	 * Add a fully-qualified path to an include-file.
+	 *
+	 * @param include the path to the include file to add
+	 */
+	public void addInclude(String include) {
+		includes.add(include);
+	}
+
+	/**
+	 * Add a path to a TRAP file (relative to the trap directory).
+	 * 
+	 * @param trap the path to the trap file to add
+	 * @return true if the path was not already present
+	 */
+	public boolean addTrap(String trap) {
+		return traps.add(trap);
+	}
+
+	/**
+	 * Check if this set contains a TRAP path
+	 * 
+	 * @param trap the path to check
+	 * @return true if this set contains the path
+	 */
+	public boolean containsTrap(String trap) {
+		return trap.contains(trap);
+	}
+
+	/**
+	 * Are the sources mentioned in this TRAP set disjoint from the given
+	 * set of paths?
+	 * 
+	 * @param paths the set of paths to check disjointness with 
+	 * @return true if and only if the paths are disjoint
+	 */
+	public boolean sourcesDisjointFrom(Set<String> paths) {
+		for (String source : sources)
+			if (paths.contains(source))
+				return false;
+		return true;
+	}
+
+	/**
+	 * Save this TRAP set to a .set file (or throw a ResourceError on failure)
+	 * 
+	 * @param file the file in which to save this set
+	 */
+	@Override
+	public void save(Path file) {
+		super.save(file);
+		this.file = file;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see java.lang.Object#toString()
+	 */
+	@Override
+	public String toString() {
+		StringBuilder sb = new StringBuilder();
+		appendHeaderString(sb, HEADER, LATEST_VERSION);
+		appendSet(sb, SOURCES, sources);
+		appendSet(sb, INCLUDES, includes);
+		appendSet(sb, OBJECTS, objects);
+		appendSet(sb, INPUT_OBJECTS, inputObjects);
+		appendSet(sb, TRAPS, traps);
+		return sb.toString();
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/NoopTransformer.java

@@ -0,0 +1,8 @@
+package com.semmle.util.trap.pathtransformers;
+
+public class NoopTransformer extends PathTransformer {
+	@Override
+	public String transform(String input) {
+		return input;
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/PathTransformer.java

@@ -0,0 +1,54 @@
+package com.semmle.util.trap.pathtransformers;
+
+import java.io.File;
+
+import com.semmle.util.files.FileUtil;
+import com.semmle.util.process.Env;
+import com.semmle.util.process.Env.Var;
+
+public abstract class PathTransformer {
+	public abstract String transform(String input);
+
+	/**
+	 * Convert a file to its path in the (code) database. Turns file paths into
+	 * canonical, absolute, strings and normalises away Unix/Windows differences.
+	 */
+	public String fileAsDatabaseString(File file) {
+		String path = file.getPath();
+		// For /!unknown-binary-location/... and /modules/...
+		// paths, on Windows the standard code wants to
+		// normalise them to e.g. C:/!unknown-binary-location/...
+		// which is particularly annoying for cross-platform test
+		// output. We therefore handle them specially here.
+		if (path.matches("^[/\\\\](!unknown-binary-location|modules)[/\\\\].*")) {
+			return path.replace('\\', '/');
+		}
+		if (Boolean.valueOf(Env.systemEnv().get(Var.SEMMLE_PRESERVE_SYMLINKS)))
+			path = FileUtil.simplifyPath(file);
+		else
+			path = FileUtil.tryMakeCanonical(file).getPath();
+		return transform(FileUtil.normalisePath(path));
+	}
+
+	/**
+	 * Utility method for extractors: Canonicalise the given path as required
+	 * for the current extraction. Unlike {@link FileUtil#tryMakeCanonical(File)},
+	 * this method is consistent with {@link #fileAsDatabaseString(File)}.
+	 */
+	public File canonicalFile(String path) {
+		return new File(fileAsDatabaseString(new File(path)));
+	}
+
+	private static final PathTransformer DEFAULT_TRANSFORMER;
+	static {
+		String layout = Env.systemEnv().get(Var.SEMMLE_PATH_TRANSFORMER);
+		if (layout == null)
+			DEFAULT_TRANSFORMER = new NoopTransformer();
+		else
+			DEFAULT_TRANSFORMER = new ProjectLayoutTransformer(new File(layout));
+	}
+
+	public static PathTransformer std() {
+		return DEFAULT_TRANSFORMER;
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/ProjectLayoutTransformer.java

@@ -0,0 +1,37 @@
+package com.semmle.util.trap.pathtransformers;
+
+import java.io.File;
+
+import com.semmle.util.projectstructure.ProjectLayout;
+
+public class ProjectLayoutTransformer extends PathTransformer {
+	private final ProjectLayout layout;
+	
+	public ProjectLayoutTransformer(File file) {
+		layout = new ProjectLayout(file);
+	}
+	
+	@Override
+	public String transform(String input) {
+		if (isWindowsPath(input, 0)) {
+			String result = layout.artificialPath('/' + input);
+			if (result == null) {
+				return input;
+			} else if (isWindowsPath(result, 1) && result.charAt(0) == '/') {
+				return result.substring(1);
+			} else {
+				return result;
+			}
+		} else {
+			String result = layout.artificialPath(input);
+			return result != null ? result : input;
+		}
+	}
+	
+	private static boolean isWindowsPath(String s, int startAt) {
+		return s.length() >= (3 + startAt) &&
+				s.charAt(startAt) != '/' &&
+				s.charAt(startAt + 1) == ':' &&
+				s.charAt(startAt + 2) == '/';
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/unicode/UTF8Util.java

@@ -0,0 +1,52 @@
+package com.semmle.util.unicode;
+
+public class UTF8Util {
+	/**
+	 * Get the length (in Unicode code units, not code points) of the longest prefix of
+	 * a string that can be UTF-8 encoded in no more than the given number of bytes.
+	 *
+	 * <p>
+	 * Unencodable characters (such as lone surrogate halves or low surrogates
+	 * that do not follow a high surrogate) are treated as being encoded in
+	 * three bytes. This is safe since on encoding they will be replaced by
+	 * a replacement character, which in turn will take at most three bytes to
+	 * encode.
+	 * </p>
+	 *
+	 * @param str string to encode
+	 * @param maxEncodedLength maximum number of bytes for the encoded prefix
+	 * @return length of the prefix
+	 */
+	public static int encodablePrefixLength(String str, int maxEncodedLength) {
+		// no character takes more than three bytes to encode
+		if (str.length() > maxEncodedLength / 3) {
+			int encodedLength = 0;
+			for (int i = 0; i < str.length(); ++i) {
+				int oldI = i;
+				char c = str.charAt(i);
+				if (c <= 0x7f) {
+					encodedLength += 1;
+				} else if (c <= 0x7ff) {
+					encodedLength += 2;
+				} else if (Character.isHighSurrogate(c)) {
+					// surrogate pairs take four bytes to encode
+					if (i+1 < str.length() && Character.isLowSurrogate(str.charAt(i+1))) {
+						encodedLength += 4;
+						++i;
+					} else {
+						// lone high surrogate, assume length three
+						encodedLength += 3;
+					}
+				} else {
+					encodedLength += 3;
+				}
+
+				if (encodedLength > maxEncodedLength) {
+					return oldI;
+				}
+			}
+		}
+
+		return str.length();
+	}
+}

java/kotlin-extractor2/src/main/java/com/semmle/util/zip/MultiMemberGZIPInputStream.java

@@ -0,0 +1,71 @@
+package com.semmle.util.zip;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PushbackInputStream;
+import java.util.zip.GZIPInputStream;
+
+public class MultiMemberGZIPInputStream extends GZIPInputStream {
+
+	public MultiMemberGZIPInputStream(InputStream in, int size) throws IOException {
+		// Wrap the stream in a PushbackInputStream...
+		super(new PushbackInputStream(in, size), size);
+		this.size = size;
+	}
+
+	public MultiMemberGZIPInputStream(InputStream in) throws IOException {
+		// Wrap the stream in a PushbackInputStream...
+		super(new PushbackInputStream(in, 1024));
+		this.size = -1;
+	}
+
+	private MultiMemberGZIPInputStream child;
+	private int size;
+	private boolean eos;
+
+	@Override
+	public int read(byte[] inputBuffer, int inputBufferOffset, int inputBufferLen) throws IOException {
+		if (eos) {
+			return -1;
+		}
+		else if (child != null) {
+			return child.read(inputBuffer, inputBufferOffset, inputBufferLen);
+		}
+		int charsRead = super.read(inputBuffer, inputBufferOffset, inputBufferLen);
+		if (charsRead == -1) {
+			// Push any remaining buffered data back onto the stream
+			// If the stream is then not empty, use it to construct
+			// a new instance of this class and delegate this and any
+			// future calls to it...
+			int n = inf.getRemaining() - 8;
+			if (n > 0) {
+				// More than 8 bytes remaining in deflater
+				// First 8 are gzip trailer. Add the rest to
+				// any un-read data...
+				((PushbackInputStream) this.in).unread(buf, len - n, n);
+			} else {
+				// Nothing in the buffer. We need to know whether or not
+				// there is unread data available in the underlying stream
+				// since the base class will not handle an empty file.
+				// Read a byte to see if there is data and if so,
+				// push it back onto the stream...
+				byte[] b = new byte[1];
+				int ret = in.read(b, 0, 1);
+				if (ret == -1) {
+					eos = true;
+					return -1;
+				} else {
+					((PushbackInputStream) this.in).unread(b, 0, 1);
+				}
+			}
+			if(size == -1)
+				child = new MultiMemberGZIPInputStream(in);
+			else
+				child = new MultiMemberGZIPInputStream(in, size);
+			return child.read(inputBuffer, inputBufferOffset, inputBufferLen);
+		} else {
+			return charsRead;
+		}
+	}
+	
+}

java/kotlin-extractor2/src/main/kotlin/ExternalDeclExtractor.kt

@@ -0,0 +1,215 @@
+package com.github.codeql
+
+import com.semmle.extractor.java.OdasaOutput
+import com.semmle.util.data.StringDigestor
+import org.jetbrains.kotlin.analysis.api.KaSession
+import org.jetbrains.kotlin.analysis.api.symbols.*
+import org.jetbrains.kotlin.analysis.api.symbols.markers.KaNamedSymbol
+import java.io.BufferedWriter
+import java.io.File
+import java.lang.Error
+import java.util.ArrayList
+import java.util.HashSet
+
+context (KaSession)
+class ExternalDeclExtractor(
+    val logger: FileLogger,
+    val compression: Compression,
+    val invocationTrapFile: String,
+    val sourceFilePath: String,
+    /* OLD: KE1
+    val primitiveTypeMapping: PrimitiveTypeMapping,
+    val pluginContext: IrPluginContext,
+    val globalExtensionState: KotlinExtractorGlobalState,
+     */
+    val diagnosticTrapWriter: DiagnosticTrapWriter
+) {
+
+    val declBinaryNames = HashMap<KaSymbol, String>()
+    val externalDeclsDone = HashSet<Pair<String, String>>()
+    val externalDeclWorkList = ArrayList<Pair<KaSymbol, String>>()
+
+    val propertySignature = ";property"
+    val fieldSignature = ";field"
+
+    val output =
+        OdasaOutput(false, compression, logger).also {
+            it.setCurrentSourceFile(File(sourceFilePath))
+        }
+
+    fun extractLater(d: KaSymbol, signature: String): Boolean {
+        /* OLD: KE1
+        if (d !is IrClass && !isExternalFileClassMember(d)) {
+            logger.errorElement(
+                "External declaration is neither a class, nor a top-level declaration",
+                d
+            )
+            return false
+        }
+        */
+        val declBinaryName = declBinaryNames.getOrPut(d) { getSymbolBinaryName(d) }
+        val ret = externalDeclsDone.add(Pair(declBinaryName, signature))
+        if (ret) externalDeclWorkList.add(Pair(d, signature))
+        return ret
+    }
+
+    fun extractLater(c: KaClassSymbol) = extractLater(c, "")
+
+    fun writeStubTrapFile(e: KaSymbol, signature: String = "") {
+        extractElement(e, signature, true) { trapFileBW, _, _ ->
+            trapFileBW.write(
+                "// Trap file stubbed because this declaration was extracted from source in $sourceFilePath\n"
+            )
+            trapFileBW.write("// Part of invocation $invocationTrapFile\n")
+        }
+    }
+
+    private fun extractElement(
+        element: KaSymbol,
+        possiblyLongSignature: String,
+        fromSource: Boolean,
+        extractorFn: (BufferedWriter, String, OdasaOutput.TrapFileManager) -> Unit
+    ) {
+        // In order to avoid excessively long signatures which can lead to trap file names longer
+        // than the filesystem
+        // limit, we truncate and add a hash to preserve uniqueness if necessary.
+        val signature =
+            if (possiblyLongSignature.length > 100) {
+                possiblyLongSignature.substring(0, 92) +
+                    "#" +
+                    StringDigestor.digest(possiblyLongSignature).substring(0, 8)
+            } else {
+                possiblyLongSignature
+            }
+        output.getTrapLockerForDecl(element, signature, fromSource).useAC { locker ->
+            locker.trapFileManager.useAC { manager ->
+                val shortName =
+                    when (element) {
+                        is KaNamedSymbol -> element.name.asString()
+                        is KaFileSymbol -> "(TODO file symbol name)"
+                        else -> "(unknown name)"
+                    }
+                if (manager == null) {
+                    logger.info("Skipping extracting external decl $shortName")
+                } else {
+                    val trapFile = manager.file
+                    logger.info("Will write TRAP file $trapFile")
+                    val trapTmpFile =
+                        File.createTempFile(
+                            "${trapFile.nameWithoutExtension}.",
+                            ".${trapFile.extension}.tmp",
+                            trapFile.parentFile
+                        )
+                    logger.debug("Writing temporary TRAP file $trapTmpFile")
+                    try {
+                        compression.bufferedWriter(trapTmpFile).use {
+                            extractorFn(it, signature, manager)
+                        }
+
+                        if (!trapTmpFile.renameTo(trapFile)) {
+                            logger.error("Failed to rename $trapTmpFile to $trapFile")
+                        }
+                        logger.info("Finished writing TRAP file $trapFile")
+                    } catch (e: Throwable) {
+                        manager.setHasError()
+                        logger.error(
+                            "Failed to extract '$shortName'. Partial TRAP file location is $trapTmpFile",
+                            e
+                        )
+                    }
+                }
+            }
+        }
+    }
+
+    fun extractExternalClasses() {
+        do {
+            val nextBatch = ArrayList(externalDeclWorkList)
+            externalDeclWorkList.clear()
+            nextBatch.forEach { workPair ->
+                val (sym, possiblyLongSignature) = workPair
+                extractElement(sym, possiblyLongSignature, false) {
+                    trapFileBW,
+                    signature,
+                    manager ->
+                    val binaryPath = getSymbolBinaryPath(sym)
+                    if (binaryPath == null) {
+                        sym.psi?.also {
+                            logger.errorElement("Unable to get binary path", it)
+                        } ?: logger.error("Unable to get binary path")
+                    } else {
+                        // We want our comments to be the first thing in the file,
+                        // so start off with a PlainTrapWriter
+                        val tw =
+                            PlainTrapWriter(
+                                logger.loggerBase,
+                                TrapLabelManager(),
+                                trapFileBW,
+                                diagnosticTrapWriter
+                            )
+                        tw.writeComment(
+                            "Generated by the CodeQL Kotlin extractor for external dependencies"
+                        )
+                        tw.writeComment("Part of invocation $invocationTrapFile")
+                        if (signature != possiblyLongSignature) {
+                            tw.writeComment(
+                                "Function signature abbreviated; full signature is: $possiblyLongSignature"
+                            )
+                        }
+                        // Now elevate to a SourceFileTrapWriter, and populate the
+                        // file information if needed:
+                        val ftw = tw.makeFileTrapWriter(binaryPath, true)
+
+                        val fileExtractor =
+                            KotlinFileExtractor(
+                                logger,
+                                ftw,
+                                this,
+                                /* OLD: KE1
+                                null,
+                                binaryPath,
+                                manager,
+                                primitiveTypeMapping,
+                                pluginContext,
+                                KotlinFileExtractor.DeclarationStack(),
+                                globalExtensionState
+                                */
+                            )
+
+                        if (sym is KaClassSymbol) {
+                            // Populate a location and compilation-unit package for the file. This
+                            // is similar to
+                            // the beginning of `KotlinFileExtractor.extractFileContents` but
+                            // without an `IrFile`
+                            // to start from.
+                            val pkg = sym.classId?.packageFqName?.asString() ?: ""
+                            val pkgId = fileExtractor.extractPackage(pkg)
+                            ftw.writeHasLocation(ftw.fileId, ftw.getWholeFileLocation())
+                            ftw.writeCupackage(ftw.fileId, pkgId)
+
+                            fileExtractor.extractClassSource(
+                                sym,
+                                extractDeclarations = /* OLD: KE1 !sym.isFileClass */true,
+                                /* OLD: KE1
+                                extractStaticInitializer = false,
+                                extractPrivateMembers = false,
+                                extractFunctionBodies = false
+                                */
+                            )
+                        } else {
+                            fileExtractor.extractDeclaration(
+                                sym as KaDeclarationSymbol,
+                                /* OLD: KE1
+                                extractPrivateMembers = false,
+                                extractFunctionBodies = false,
+                                extractAnnotations = true
+                                */
+                            )
+                        }
+                    }
+                }
+            }
+        } while (externalDeclWorkList.isNotEmpty())
+        output.writeTrapSet()
+    }
+}

java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt

@@ -0,0 +1,551 @@
+package com.github.codeql
+
+import com.intellij.openapi.editor.Document
+import com.intellij.psi.impl.DebugUtil
+import com.intellij.psi.PsiFile
+import com.semmle.util.files.FileUtil
+import com.semmle.util.trap.pathtransformers.PathTransformer
+import java.io.File
+import java.io.FileOutputStream
+import java.nio.file.Files
+import java.nio.file.Paths
+import java.util.concurrent.Executors
+import kotlinx.coroutines.asCoroutineDispatcher
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
+import kotlinx.coroutines.withContext
+import org.jetbrains.kotlin.analysis.api.KaAnalysisApiInternals
+import org.jetbrains.kotlin.analysis.api.KaSession
+import org.jetbrains.kotlin.analysis.api.analyze
+import org.jetbrains.kotlin.analysis.api.platform.lifetime.KotlinAlwaysAccessibleLifetimeTokenFactory
+import org.jetbrains.kotlin.analysis.api.platform.lifetime.KotlinLifetimeTokenFactory
+import org.jetbrains.kotlin.analysis.api.projectStructure.KaSourceModule
+import org.jetbrains.kotlin.analysis.api.standalone.buildStandaloneAnalysisAPISession
+import org.jetbrains.kotlin.analysis.project.structure.builder.buildKtLibraryModule
+import org.jetbrains.kotlin.analysis.project.structure.builder.buildKtSdkModule
+import org.jetbrains.kotlin.analysis.project.structure.builder.buildKtSourceModule
+import org.jetbrains.kotlin.cli.common.arguments.parseCommandLineArguments
+import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
+import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.platform.jvm.JvmPlatforms
+import org.jetbrains.kotlin.psi.*
+
+fun main(args: Array<String>) {
+    System.out.println("Kotlin Extractor 2 running")
+    try {
+        runExtractor(args)
+        // We catch Throwable rather than Exception, as we want to
+        // log about the failure even if we get a stack overflow
+        // or an assertion failure in one file.
+    } catch (e: Throwable) {
+        // If we get an exception at the top level, then something's
+        // gone very wrong. Don't try to be too fancy, but try to
+        // log a simple message.
+        val msg = "[ERROR] CodeQL Kotlin extractor: Top-level exception."
+        // First, if we can find our log directory, then let's try
+        // making a log file there:
+        val extractorLogDir = System.getenv("CODEQL_EXTRACTOR_JAVA_LOG_DIR")
+        if (extractorLogDir != null && extractorLogDir != "") {
+            // We use a slightly different filename pattern compared
+            // to normal logs. Just the existence of a `-top` log is
+            // a sign that something's gone very wrong.
+            val logFile = File.createTempFile("kotlin-extractor-top.", ".log", File(extractorLogDir))
+            logFile.writeText(msg)
+            // Now we've got that out, let's see if we can append a stack trace too
+            logFile.appendText(e.stackTraceToString())
+        } else {
+            // We don't have much choice here except to print to
+            // stderr and hope for the best.
+            System.err.println(msg)
+            e.printStackTrace(System.err)
+        }
+    }
+}
+
+@OptIn(KaAnalysisApiInternals::class)
+private fun runExtractor(args: Array<String>) {
+    val startTimeMs = System.currentTimeMillis()
+
+    val invocationTrapFile = args[0]
+    val kotlinArgs = args.drop(1)
+    System.out.println("Invocation TRAP file: " + invocationTrapFile)
+
+    // This default should be kept in sync with
+    // com.semmle.extractor.java.interceptors.KotlinInterceptor.initializeExtractionContext
+    val trapDir =
+        File(
+            System.getenv("CODEQL_EXTRACTOR_JAVA_TRAP_DIR").takeUnless { it.isNullOrEmpty() }
+                ?: "kotlin-extractor/trap"
+        )
+
+    // The invocation TRAP file will already have been started
+    // as an uncompressed TRAP file before the extractor is run,
+    // so we always use no compression and we open it in append mode.
+    FileOutputStream(File(invocationTrapFile), true).bufferedWriter().use { invocationTrapFileBW
+        ->
+/*
+OLD: KE1
+        val usesK2 = usesK2(pluginContext)
+*/
+        val invocationExtractionProblems = ExtractionProblems()
+        val invocationLabelManager = TrapLabelManager()
+        val diagnosticCounter = DiagnosticCounter()
+        val loggerBase = LoggerBase(diagnosticCounter)
+        val dtw = DiagnosticTrapWriter(loggerBase, invocationLabelManager, invocationTrapFileBW)
+        // The diagnostic TRAP file has already defined #compilation = *
+        val compilation: Label<DbCompilation> = StringLabel("compilation")
+        dtw.writeCompilation_started(compilation)
+        /*
+        OLD: KE1
+                dtw.writeCompilation_info(
+                    compilation,
+                    "Kotlin Compiler Version",
+                    KotlinCompilerVersion.getVersion() ?: "<unknown>"
+                )
+                val extractor_name =
+                    this::class.java.getResource("extractor.name")?.readText() ?: "<unknown>"
+                dtw.writeCompilation_info(compilation, "Kotlin Extractor Name", extractor_name)
+                dtw.writeCompilation_info(compilation, "Uses Kotlin 2", usesK2.toString())
+                if (compilationStartTime != null) {
+                    dtw.writeCompilation_compiler_times(
+                        compilation,
+                        -1.0,
+                        (System.currentTimeMillis() - compilationStartTime) / 1000.0
+                    )
+                }
+        */
+        dtw.flush()
+        val logger = Logger(loggerBase, dtw)
+        logger.info("Extraction started")
+        logger.flush()
+        logger.infoVerbosity()
+        logger.info("Extraction for invocation TRAP file $invocationTrapFile")
+        logger.flush()
+        /*
+        OLD: KE1
+                logger.info("Kotlin version ${KotlinCompilerVersion.getVersion()}")
+                logger.flush()
+                logPeakMemoryUsage(logger, "before extractor")
+        */
+        val compression = getCompression(logger)
+
+        /*
+        OLD: KE1
+                val primitiveTypeMapping = PrimitiveTypeMapping(logger, pluginContext)
+                // FIXME: FileUtil expects a static global logger
+                // which should be provided by SLF4J's factory facility. For now we set it here.
+                FileUtil.logger = logger
+        */
+        val srcDir =
+            File(
+                System.getenv("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR").takeUnless {
+                    it.isNullOrEmpty()
+                } ?: "kotlin-extractor/src"
+            )
+        srcDir.mkdirs()
+        /*
+        OLD: KE1
+                val globalExtensionState = KotlinExtractorGlobalState()
+        */
+        doAnalysis(compression, trapDir, srcDir, loggerBase, dtw, compilation, invocationExtractionProblems, kotlinArgs, invocationTrapFile)
+        loggerBase.printLimitedDiagnosticCounts(dtw)
+        /*
+        OLD: KE1
+                logPeakMemoryUsage(logger, "after extractor")
+        */
+        logger.info("Extraction completed")
+        logger.flush()
+        val compilationTimeMs = System.currentTimeMillis() - startTimeMs
+        dtw.writeCompilation_finished(
+            compilation,
+            -1.0,
+            compilationTimeMs.toDouble() / 1000,
+            invocationExtractionProblems.extractionResult()
+        )
+        dtw.flush()
+        loggerBase.close()
+        System.exit(0) // TODO: figure out what's keeping the JVM awake
+    }
+}
+
+private fun doAnalysis(
+    compression: Compression,
+    trapDir: File,
+    srcDir: File,
+    loggerBase: LoggerBase,
+    dtw: DiagnosticTrapWriter,
+    compilation: Label<DbCompilation>,
+    invocationExtractionProblems: ExtractionProblems,
+    args: List<String>,
+    invocationTrapFile: String
+) {
+    lateinit var sourceModule: KaSourceModule
+    val k2args: K2JVMCompilerArguments = parseCommandLineArguments(args.toList())
+
+    // TODO: Collect the messages, and log them?
+    val ourLanguageVersionSettings = k2args.toLanguageVersionSettings(MessageCollector.NONE)
+
+    val session = buildStandaloneAnalysisAPISession {
+        registerProjectService(KotlinLifetimeTokenFactory::class.java, KotlinAlwaysAccessibleLifetimeTokenFactory())
+
+        // TODO: Is there a better way we can do all this directly from k2args?
+
+        buildKtModuleProvider {
+            platform = JvmPlatforms.defaultJvmPlatform
+            val sdk = addModule(
+                buildKtSdkModule {
+                    addBinaryRootsFromJdkHome(Paths.get(System.getProperty("java.home")), isJre = true)
+                    addBinaryRootsFromJdkHome(Paths.get(System.getProperty("java.home")), isJre = false)
+                    platform = JvmPlatforms.defaultJvmPlatform
+                    libraryName = "JDK"
+                }
+            )
+            val lib = addModule(
+                buildKtLibraryModule {
+                    val classpath = k2args.classpath
+                    if (classpath != null) {
+                        for (cpEntry in classpath.split(File.pathSeparatorChar)) {
+                            addBinaryRoot(Paths.get(cpEntry))
+                        }
+                    }
+                    platform = JvmPlatforms.defaultJvmPlatform
+                    libraryName = "Library for classpath"
+                }
+            )
+            sourceModule = addModule(
+                buildKtSourceModule {
+                    languageVersionSettings = ourLanguageVersionSettings
+                    addSourceRoots(k2args.freeArgs.map { Paths.get(it) })
+                    addRegularDependency(sdk)
+                    addRegularDependency(lib)
+                    platform = JvmPlatforms.defaultJvmPlatform
+                    moduleName = "<source>"
+                }
+            )
+        }
+    }
+
+    val checkTrapIdentical = false // TODO
+
+    analyze(sourceModule) {
+        val maxThreads = 8 // TODO: Later, default to $CODEQL_THREADS or Runtime.getRuntime().availableProcessors()
+        // If a Kotlin coroutine yields, then a thread will be freed up
+        // and start extracting the next file. We want to avoid having
+        // lots of TRAP files open at once, so we use a semaphore so that
+        // we only have `maxThreads` coroutines with an open TRAP file
+        // at once.
+        val extractorThreads = Semaphore(maxThreads)
+        Executors.newFixedThreadPool(maxThreads).asCoroutineDispatcher().use { dispatcher ->
+
+            runBlocking {
+                withContext(dispatcher) {
+                    val psiFiles = session.modulesWithFiles.getValue(sourceModule)
+                    var fileNumber = 0
+                    val dump_psi = System.getenv("CODEQL_EXTRACTOR_JAVA_KOTLIN_DUMP") == "true"
+                    for (psiFile in psiFiles) {
+                        val thisFileNumber = fileNumber++
+                        launch {
+                            extractorThreads.withPermit {
+                                if (psiFile is KtFile) {
+                                    if (dump_psi) {
+                                        val showWhitespaces = false
+                                        val showRanges = true
+                                        loggerBase.info(dtw, DebugUtil.psiToString(psiFile, showWhitespaces, showRanges))
+                                    }
+                                    val fileExtractionProblems = FileExtractionProblems(invocationExtractionProblems)
+                                    try {
+                                        val fileDiagnosticTrapWriter = dtw.makeSourceFileTrapWriter(psiFile, true)
+                                        fileDiagnosticTrapWriter.writeCompilation_compiling_files(
+                                            compilation,
+                                            thisFileNumber,
+                                            fileDiagnosticTrapWriter.fileId
+                                        )
+                                        doFile(
+                                            thisFileNumber,
+                                            compression,
+                                            /*
+                                            OLD: KE1
+                                            fileExtractionProblems,
+                                            */
+                                            invocationTrapFile,
+                                            fileDiagnosticTrapWriter,
+                                            loggerBase,
+                                            checkTrapIdentical,
+                                            trapDir,
+                                            srcDir,
+                                            psiFile,
+                                            /*
+                                            OLD: KE1
+                                            primitiveTypeMapping,
+                                            pluginContext,
+                                            globalExtensionState
+                                            */
+                                        )
+                                        fileDiagnosticTrapWriter.writeCompilation_compiling_files_completed(
+                                            compilation,
+                                            thisFileNumber,
+                                            fileExtractionProblems.extractionResult()
+                                        )
+                                        // We catch Throwable rather than Exception, as we want to
+                                        // continue trying to extract everything else even if we get a
+                                        // stack overflow or an assertion failure in one file.
+                                    } catch (e: Throwable) {
+                                        fileExtractionProblems.setNonRecoverableProblem()
+                                        loggerBase.error(dtw, "Extraction failed while extracting '${psiFile.virtualFilePath}'.", e)
+                                    }
+                                } else {
+                                    System.out.println("Warning: Not a KtFile")
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+/*
+OLD: KE1
+import com.github.codeql.utils.versions.usesK2
+import java.io.BufferedInputStream
+import java.io.BufferedOutputStream
+import java.io.BufferedReader
+import java.io.BufferedWriter
+import java.io.File
+import java.io.FileInputStream
+import java.io.InputStreamReader
+import java.io.OutputStreamWriter
+import java.lang.management.*
+import java.util.zip.GZIPInputStream
+import java.util.zip.GZIPOutputStream
+import kotlin.system.exitProcess
+import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
+import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
+import org.jetbrains.kotlin.config.KotlinCompilerVersion
+import org.jetbrains.kotlin.ir.declarations.*
+import org.jetbrains.kotlin.ir.util.*
+
+class KotlinExtractorExtension(
+    // The filepath for the invocation TRAP file.
+    // This TRAP file is for this invocation of the extractor as a
+    // whole, not tied to a particular source file. It contains
+    // information about which files this invocation compiled, and
+    // any warnings or errors encountered during the invocation.
+    private val invocationTrapFile: String,
+    // By default, if a TRAP file we want to generate for a source
+    // file already exists, then we will do nothing. If this is set,
+    // then we will instead generate the TRAP file, and give a
+    // warning if we would generate different TRAP to that which
+    // already exists.
+    private val checkTrapIdentical: Boolean,
+    // If non-null, then this is the number of milliseconds since
+    // midnight, January 1, 1970 UTC (as returned by Java's
+    // `System.currentTimeMillis()`. If this is given, then it is used
+    // to record the time taken to compile the source code, which is
+    // presumed to be the difference between this time and the time
+    // that this plugin is invoked.
+    private val compilationStartTime: Long?,
+    // Under normal conditions, the extractor runs during a build of
+    // the project, and kotlinc continues after the plugin has finished.
+    // If the plugin is being used independently of a build, then this
+    // can be set to true to make the plugin terminate the kotlinc
+    // invocation when it has finished. This means that kotlinc will not
+    // write any `.class` files etc.
+    private val exitAfterExtraction: Boolean
+) : IrGenerationExtension {
+
+    private fun logPeakMemoryUsage(logger: Logger, time: String) {
+        logger.info("Peak memory: Usage $time")
+
+        val beans = ManagementFactory.getMemoryPoolMXBeans()
+        var heap: Long = 0
+        var nonheap: Long = 0
+        for (bean in beans) {
+            val peak = bean.getPeakUsage().getUsed()
+            val kind =
+                when (bean.getType()) {
+                    MemoryType.HEAP -> {
+                        heap += peak
+                        "heap"
+                    }
+                    MemoryType.NON_HEAP -> {
+                        nonheap += peak
+                        "non-heap"
+                    }
+                    else -> "unknown"
+                }
+            logger.info("Peak memory: * Peak for $kind bean ${bean.getName()} is $peak")
+        }
+        logger.info("Peak memory: * Total heap peak: $heap")
+        logger.info("Peak memory: * Total non-heap peak: $nonheap")
+    }
+}
+
+class KotlinExtractorGlobalState {
+    // These three record mappings of classes, functions and fields that should be replaced wherever
+    // they are found.
+    // As of now these are only used to fix IR generated by the Gradle Android Extensions plugin,
+    // hence e.g. IrProperty
+    // doesn't have a map as that plugin doesn't generate them. If and when these are used more
+    // widely additional maps
+    // should be added here.
+    val syntheticToRealClassMap = HashMap<IrClass, IrClass?>()
+    val syntheticToRealFunctionMap = HashMap<IrFunction, IrFunction?>()
+    val syntheticToRealFieldMap = HashMap<IrField, IrField?>()
+    val syntheticRepeatableAnnotationContainers = HashMap<IrClass, IrClass>()
+}
+*/
+
+/*
+The `ExtractionProblems` class is used to record whether this invocation
+had any problems. It distinguish 2 kinds of problem:
+* Recoverable problems: e.g. if we check something that we expect to be
+  non-null and find that it is null.
+* Non-recoverable problems: if we catch an exception.
+*/
+open class ExtractionProblems {
+    private var recoverableProblem = false
+    private var nonRecoverableProblem = false
+
+    open fun setRecoverableProblem() {
+        recoverableProblem = true
+    }
+
+    open fun setNonRecoverableProblem() {
+        nonRecoverableProblem = true
+    }
+
+    fun extractionResult(): Int {
+        if (nonRecoverableProblem) {
+            return 2
+        } else if (recoverableProblem) {
+            return 1
+        } else {
+            return 0
+        }
+    }
+}
+
+/*
+The `FileExtractionProblems` is analogous to `ExtractionProblems`,
+except it records whether there were any problems while extracting a
+particular source file.
+*/
+class FileExtractionProblems(val invocationExtractionProblems: ExtractionProblems) :
+    ExtractionProblems() {
+    override fun setRecoverableProblem() {
+        super.setRecoverableProblem()
+        invocationExtractionProblems.setRecoverableProblem()
+    }
+
+    override fun setNonRecoverableProblem() {
+        super.setNonRecoverableProblem()
+        invocationExtractionProblems.setNonRecoverableProblem()
+    }
+}
+
+context (KaSession)
+private fun doFile(
+    fileNumber: Int,
+    compression: Compression,
+    /*
+    OLD: KE1
+        fileExtractionProblems: FileExtractionProblems,
+     */
+    invocationTrapFile: String,
+    fileDiagnosticTrapWriter: FileTrapWriter,
+    loggerBase: LoggerBase,
+    checkTrapIdentical: Boolean,
+    dbTrapDir: File,
+    dbSrcDir: File,
+    srcFile: KtFile,
+    /*
+    OLD: KE1
+        primitiveTypeMapping: PrimitiveTypeMapping,
+        pluginContext: IrPluginContext,
+        globalExtensionState: KotlinExtractorGlobalState
+    */
+) {
+    val srcFilePath = srcFile.virtualFilePath
+    val logger = FileLogger(loggerBase, fileDiagnosticTrapWriter, fileNumber)
+    logger.info("Extracting file $srcFilePath")
+    logger.flush()
+
+    val srcFileRelativePath = PathTransformer.std().fileAsDatabaseString(File(srcFilePath))
+
+    val dbSrcFilePath = FileUtil.appendAbsolutePath(dbSrcDir, srcFileRelativePath).toPath()
+    val dbSrcDirPath = dbSrcFilePath.parent
+    Files.createDirectories(dbSrcDirPath)
+    val srcTmpFile =
+        File.createTempFile(
+            dbSrcFilePath.fileName.toString() + ".",
+            ".src.tmp",
+            dbSrcDirPath.toFile()
+        )
+    srcTmpFile.outputStream().use { Files.copy(Paths.get(srcFilePath), it) }
+    srcTmpFile.renameTo(dbSrcFilePath.toFile())
+
+    val trapFileName = FileUtil.appendAbsolutePath(dbTrapDir, "$srcFileRelativePath.trap").getAbsolutePath()
+    val trapFileWriter = getTrapFileWriter(compression, logger, checkTrapIdentical, trapFileName)
+
+    trapFileWriter.run { trapFileBW ->
+        // We want our comments to be the first thing in the file,
+        // so start off with a PlainTrapWriter
+        val tw =
+            PlainTrapWriter(
+                logger,
+                TrapLabelManager(),
+                trapFileBW,
+                fileDiagnosticTrapWriter.getDiagnosticTrapWriter()
+            )
+        tw.writeComment("Generated by the CodeQL Kotlin extractor for kotlin source code")
+        /*
+        OLD: KE1
+                tw.writeComment("Part of invocation $invocationTrapFile")
+        */
+        // Now elevate to a SourceFileTrapWriter, and populate the
+        // file information
+        val sftw = tw.makeSourceFileTrapWriter(srcFile, true)
+        val externalDeclExtractor =
+            ExternalDeclExtractor(
+                logger,
+                compression,
+                invocationTrapFile,
+                srcFilePath,
+                /*
+                OLD: KE1
+                                primitiveTypeMapping,
+                                pluginContext,
+                                globalExtensionState,
+                 */
+                fileDiagnosticTrapWriter.getDiagnosticTrapWriter()
+            )
+        /* OLD: KE1
+                                val linesOfCode = LinesOfCode(logger, sftw, srcFile)
+         */
+
+        val fileExtractor =
+            KotlinFileExtractor(
+                logger,
+                sftw,
+                externalDeclExtractor,
+                /*
+                OLD: KE1
+                                linesOfCode,
+                                srcFilePath,
+                                null,
+
+                                primitiveTypeMapping,
+                                pluginContext,
+                                KotlinFileExtractor.DeclarationStack(),
+                                globalExtensionState
+                */
+            )
+
+        fileExtractor.extractFileContents(srcFile, sftw.fileId)
+        externalDeclExtractor.extractExternalClasses()
+    }
+}

java/kotlin-extractor2/src/main/kotlin/KotlinExtractorCommandLineProcessor.kt

@@ -0,0 +1,103 @@
+package com.github.codeql
+
+/*
+OLD: KE1
+import org.jetbrains.kotlin.compiler.plugin.AbstractCliOption
+import org.jetbrains.kotlin.compiler.plugin.CliOption
+import org.jetbrains.kotlin.compiler.plugin.CommandLineProcessor
+import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.config.CompilerConfigurationKey
+
+@OptIn(ExperimentalCompilerApi::class)
+class KotlinExtractorCommandLineProcessor : CommandLineProcessor {
+    override val pluginId = "kotlin-extractor"
+
+    override val pluginOptions =
+        listOf(
+            CliOption(
+                optionName = OPTION_INVOCATION_TRAP_FILE,
+                valueDescription = "Invocation TRAP file",
+                description = "Extractor will append invocation-related TRAP to this file",
+                required = true,
+                allowMultipleOccurrences = false
+            ),
+            CliOption(
+                optionName = OPTION_CHECK_TRAP_IDENTICAL,
+                valueDescription = "Check whether different invocations produce identical TRAP",
+                description = "Check whether different invocations produce identical TRAP",
+                required = false,
+                allowMultipleOccurrences = false
+            ),
+            CliOption(
+                optionName = OPTION_COMPILATION_STARTTIME,
+                valueDescription = "The start time of the compilation as a Unix timestamp",
+                description = "The start time of the compilation as a Unix timestamp",
+                required = false,
+                allowMultipleOccurrences = false
+            ),
+            CliOption(
+                optionName = OPTION_EXIT_AFTER_EXTRACTION,
+                valueDescription =
+                    "Specify whether to call exitProcess after the extraction has completed",
+                description =
+                    "Specify whether to call exitProcess after the extraction has completed",
+                required = false,
+                allowMultipleOccurrences = false
+            )
+        )
+
+    override fun processOption(
+        option: AbstractCliOption,
+        value: String,
+        configuration: CompilerConfiguration
+    ) =
+        when (option.optionName) {
+            OPTION_INVOCATION_TRAP_FILE -> configuration.put(KEY_INVOCATION_TRAP_FILE, value)
+            OPTION_CHECK_TRAP_IDENTICAL ->
+                processBooleanOption(
+                    value,
+                    OPTION_CHECK_TRAP_IDENTICAL,
+                    KEY_CHECK_TRAP_IDENTICAL,
+                    configuration
+                )
+            OPTION_EXIT_AFTER_EXTRACTION ->
+                processBooleanOption(
+                    value,
+                    OPTION_EXIT_AFTER_EXTRACTION,
+                    KEY_EXIT_AFTER_EXTRACTION,
+                    configuration
+                )
+            OPTION_COMPILATION_STARTTIME ->
+                when (val v = value.toLongOrNull()) {
+                    is Long -> configuration.put(KEY_COMPILATION_STARTTIME, v)
+                    else ->
+                        error(
+                            "kotlin extractor: Bad argument $value for $OPTION_COMPILATION_STARTTIME"
+                        )
+                }
+            else -> error("kotlin extractor: Bad option: ${option.optionName}")
+        }
+
+    private fun processBooleanOption(
+        value: String,
+        optionName: String,
+        configKey: CompilerConfigurationKey<Boolean>,
+        configuration: CompilerConfiguration
+    ) =
+        when (value) {
+            "true" -> configuration.put(configKey, true)
+            "false" -> configuration.put(configKey, false)
+            else -> error("kotlin extractor: Bad argument $value for $optionName")
+        }
+}
+
+private val OPTION_INVOCATION_TRAP_FILE = "invocationTrapFile"
+val KEY_INVOCATION_TRAP_FILE = CompilerConfigurationKey<String>(OPTION_INVOCATION_TRAP_FILE)
+private val OPTION_CHECK_TRAP_IDENTICAL = "checkTrapIdentical"
+val KEY_CHECK_TRAP_IDENTICAL = CompilerConfigurationKey<Boolean>(OPTION_CHECK_TRAP_IDENTICAL)
+private val OPTION_COMPILATION_STARTTIME = "compilationStartTime"
+val KEY_COMPILATION_STARTTIME = CompilerConfigurationKey<Long>(OPTION_COMPILATION_STARTTIME)
+private val OPTION_EXIT_AFTER_EXTRACTION = "exitAfterExtraction"
+val KEY_EXIT_AFTER_EXTRACTION = CompilerConfigurationKey<Boolean>(OPTION_EXIT_AFTER_EXTRACTION)
+*/

java/kotlin-extractor2/src/main/kotlin/KotlinExtractorComponentRegistrar.kt

@@ -0,0 +1,32 @@
+// For ComponentRegistrar
+@file:Suppress("DEPRECATION")
+
+package com.github.codeql
+
+/*
+OLD: KE1
+import com.intellij.mock.MockProject
+import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
+import org.jetbrains.kotlin.config.CompilerConfiguration
+
+class KotlinExtractorComponentRegistrar : Kotlin2ComponentRegistrar() {
+    override fun registerProjectComponents(
+        project: MockProject,
+        configuration: CompilerConfiguration
+    ) {
+        val invocationTrapFile = configuration[KEY_INVOCATION_TRAP_FILE]
+        if (invocationTrapFile == null) {
+            throw Exception("Required argument for TRAP invocation file not given")
+        }
+        IrGenerationExtension.registerExtension(
+            project,
+            KotlinExtractorExtension(
+                invocationTrapFile,
+                configuration[KEY_CHECK_TRAP_IDENTICAL] ?: false,
+                configuration[KEY_COMPILATION_STARTTIME],
+                configuration[KEY_EXIT_AFTER_EXTRACTION] ?: false
+            )
+        )
+    }
+}
+*/

java/kotlin-extractor2/src/main/kotlin/Label.kt

@@ -0,0 +1,21 @@
+package com.github.codeql
+
+/** This represents a label (`#...`) in a TRAP file. */
+interface Label<T : AnyDbType> {
+    fun <U : AnyDbType> cast(): Label<U> {
+        @Suppress("UNCHECKED_CAST") return this as Label<U>
+    }
+}
+
+/** The label `#i`, e.g. `#123`. Most labels we generate are of this form. */
+class IntLabel<T : AnyDbType>(val i: Int) : Label<T> {
+    override fun toString(): String = "#$i"
+}
+
+/**
+ * The label `#name`, e.g. `#compilation`. This is used when labels are shared between different
+ * components (e.g. when both the interceptor and the extractor need to refer to the same label).
+ */
+class StringLabel<T : AnyDbType>(val name: String) : Label<T> {
+    override fun toString(): String = "#$name"
+}

java/kotlin-extractor2/src/main/kotlin/LinesOfCode.kt

@@ -0,0 +1,153 @@
+package com.github.codeql
+
+/*
+OLD: KE1
+import com.github.codeql.utils.versions.getPsi2Ir
+import com.intellij.psi.PsiComment
+import com.intellij.psi.PsiElement
+import com.intellij.psi.PsiWhiteSpace
+import org.jetbrains.kotlin.config.KotlinCompilerVersion
+import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.declarations.*
+import org.jetbrains.kotlin.psi.KtVisitor
+
+class LinesOfCode(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) {
+    val psi2Ir =
+        getPsi2Ir().also {
+            if (it == null) {
+                logger.warn(
+                    "Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})"
+                )
+            }
+        }
+
+    fun linesOfCodeInFile(id: Label<DbFile>): Boolean {
+        if (psi2Ir == null) {
+            return false
+        }
+        val ktFile = psi2Ir.getKtFile(file)
+        if (ktFile == null) {
+            return false
+        }
+        linesOfCodeInPsi(id, ktFile, file)
+        // Even if linesOfCodeInPsi didn't manage to extract any
+        // information, if we got as far as calling it then we have
+        // PSI info for the file
+        return true
+    }
+
+    fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label<out DbSourceline>): Boolean {
+        if (psi2Ir == null) {
+            return false
+        }
+        val p = psi2Ir.findPsiElement(d, file)
+        if (p == null) {
+            return false
+        }
+        linesOfCodeInPsi(id, p, d)
+        // Even if linesOfCodeInPsi didn't manage to extract any
+        // information, if we got as far as calling it then we have
+        // PSI info for the declaration
+        return true
+    }
+
+    private fun linesOfCodeInPsi(id: Label<out DbSourceline>, root: PsiElement, e: IrElement) {
+        val document = root.getContainingFile().getViewProvider().getDocument()
+        if (document == null) {
+            logger.errorElement("Cannot find document for PSI", e)
+            tw.writeNumlines(id, 0, 0, 0)
+            return
+        }
+
+        val rootRange = root.getTextRange()
+        val rootStartOffset = rootRange.getStartOffset()
+        val rootEndOffset = rootRange.getEndOffset()
+        if (rootStartOffset < 0 || rootEndOffset < 0) {
+            // This is synthetic, or has an invalid location
+            tw.writeNumlines(id, 0, 0, 0)
+            return
+        }
+        val rootFirstLine = document.getLineNumber(rootStartOffset)
+        val rootLastLine = document.getLineNumber(rootEndOffset)
+        if (rootLastLine < rootFirstLine) {
+            logger.errorElement("PSI ends before it starts", e)
+            tw.writeNumlines(id, 0, 0, 0)
+            return
+        }
+        val numLines = 1 + rootLastLine - rootFirstLine
+        val lineContents = Array(numLines) { LineContent() }
+
+        val visitor =
+            object : KtVisitor<Unit, Unit>() {
+                override fun visitElement(element: PsiElement) {
+                    val isComment = element is PsiComment
+                    // Comments may include nodes that aren't PsiComments,
+                    // so we don't want to visit them or we'll think they
+                    // are code.
+                    if (!isComment) {
+                        element.acceptChildren(this)
+                    }
+
+                    if (element is PsiWhiteSpace) {
+                        return
+                    }
+                    // Leaf nodes are assumed to be tokens, and
+                    // therefore we count any lines that they are on.
+                    // For comments, we actually need to look at the
+                    // outermost node, as the leaves of KDocs don't
+                    // necessarily cover all lines.
+                    if (isComment || element.getChildren().size == 0) {
+                        val range = element.getTextRange()
+                        val startOffset = range.getStartOffset()
+                        val endOffset = range.getEndOffset()
+                        // The PSI doesn't seem to have anything like
+                        // the IR's UNDEFINED_OFFSET and SYNTHETIC_OFFSET,
+                        // but < 0 still seem to represent bad/unknown
+                        // locations.
+                        if (startOffset < 0 || endOffset < 0) {
+                            logger.errorElement("PSI element has negative offset", e)
+                            return
+                        }
+                        if (startOffset > endOffset) {
+                            logger.errorElement("PSI element has negative size", e)
+                            return
+                        }
+                        // We might get e.g. an import list for a file
+                        // with no imports, which claims to have start
+                        // and end offsets of 0. Anything of 0 width
+                        // we therefore just skip.
+                        if (startOffset == endOffset) {
+                            return
+                        }
+                        val firstLine = document.getLineNumber(startOffset)
+                        val lastLine = document.getLineNumber(endOffset)
+                        if (firstLine < rootFirstLine) {
+                            logger.errorElement("PSI element starts before root", e)
+                            return
+                        } else if (lastLine > rootLastLine) {
+                            logger.errorElement("PSI element ends after root", e)
+                            return
+                        }
+                        for (line in firstLine..lastLine) {
+                            val lineContent = lineContents[line - rootFirstLine]
+                            if (isComment) {
+                                lineContent.containsComment = true
+                            } else {
+                                lineContent.containsCode = true
+                            }
+                        }
+                    }
+                }
+            }
+        root.accept(visitor)
+        val code = lineContents.count { it.containsCode }
+        val comment = lineContents.count { it.containsComment }
+        tw.writeNumlines(id, numLines, code, comment)
+    }
+
+    private class LineContent {
+        var containsComment = false
+        var containsCode = false
+    }
+}
+*/

java/kotlin-extractor2/src/main/kotlin/MetaAnnotationSupport.kt

@@ -0,0 +1,516 @@
+package com.github.codeql
+
+/*
+OLD: KE1
+import com.github.codeql.utils.versions.copyParameterToFunction
+import com.github.codeql.utils.versions.createImplicitParameterDeclarationWithWrappedDescriptor
+import com.github.codeql.utils.versions.getAnnotationType
+import java.lang.annotation.ElementType
+import java.util.HashSet
+import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
+import org.jetbrains.kotlin.builtins.StandardNames
+import org.jetbrains.kotlin.config.JvmTarget
+import org.jetbrains.kotlin.descriptors.ClassKind
+import org.jetbrains.kotlin.descriptors.annotations.KotlinRetention
+import org.jetbrains.kotlin.descriptors.annotations.KotlinTarget
+import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
+import org.jetbrains.kotlin.ir.builders.declarations.addConstructor
+import org.jetbrains.kotlin.ir.builders.declarations.addGetter
+import org.jetbrains.kotlin.ir.builders.declarations.addProperty
+import org.jetbrains.kotlin.ir.builders.declarations.addValueParameter
+import org.jetbrains.kotlin.ir.builders.declarations.buildClass
+import org.jetbrains.kotlin.ir.builders.declarations.buildField
+import org.jetbrains.kotlin.ir.declarations.IrClass
+import org.jetbrains.kotlin.ir.declarations.IrConstructor
+import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
+import org.jetbrains.kotlin.ir.declarations.IrEnumEntry
+import org.jetbrains.kotlin.ir.declarations.IrProperty
+import org.jetbrains.kotlin.ir.expressions.IrClassReference
+import org.jetbrains.kotlin.ir.expressions.IrConstructorCall
+import org.jetbrains.kotlin.ir.expressions.IrGetEnumValue
+import org.jetbrains.kotlin.ir.expressions.IrVararg
+import org.jetbrains.kotlin.ir.expressions.impl.IrClassReferenceImpl
+import org.jetbrains.kotlin.ir.expressions.impl.IrConstructorCallImpl
+import org.jetbrains.kotlin.ir.expressions.impl.IrGetEnumValueImpl
+import org.jetbrains.kotlin.ir.expressions.impl.IrGetFieldImpl
+import org.jetbrains.kotlin.ir.expressions.impl.IrGetValueImpl
+import org.jetbrains.kotlin.ir.expressions.impl.IrReturnImpl
+import org.jetbrains.kotlin.ir.expressions.impl.IrVarargImpl
+import org.jetbrains.kotlin.ir.symbols.IrClassSymbol
+import org.jetbrains.kotlin.ir.types.typeWith
+import org.jetbrains.kotlin.ir.util.constructedClass
+import org.jetbrains.kotlin.ir.util.constructors
+import org.jetbrains.kotlin.ir.util.deepCopyWithSymbols
+import org.jetbrains.kotlin.ir.util.defaultType
+import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable
+import org.jetbrains.kotlin.ir.util.getAnnotation
+import org.jetbrains.kotlin.ir.util.hasAnnotation
+import org.jetbrains.kotlin.ir.util.hasEqualFqName
+import org.jetbrains.kotlin.ir.util.parentAsClass
+import org.jetbrains.kotlin.ir.util.primaryConstructor
+import org.jetbrains.kotlin.load.java.JvmAnnotationNames
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
+
+class MetaAnnotationSupport(
+    private val logger: FileLogger,
+    private val pluginContext: IrPluginContext,
+    private val extractor: KotlinFileExtractor
+) {
+
+    // Taken from AdditionalIrUtils.kt (not available in Kotlin < 1.6)
+    private val IrConstructorCall.annotationClass
+        get() = this.symbol.owner.constructedClass
+
+    // Taken from AdditionalIrUtils.kt (not available in Kotlin < 1.6)
+    private fun IrConstructorCall.isAnnotationWithEqualFqName(fqName: FqName): Boolean =
+        annotationClass.hasEqualFqName(fqName)
+
+    // Adapted from RepeatedAnnotationLowering.kt
+    fun groupRepeatableAnnotations(annotations: List<IrConstructorCall>): List<IrConstructorCall> {
+        if (annotations.size < 2) return annotations
+
+        val annotationsByClass = annotations.groupByTo(mutableMapOf()) { it.annotationClass }
+        if (annotationsByClass.values.none { it.size > 1 }) return annotations
+
+        val result = mutableListOf<IrConstructorCall>()
+        for (annotation in annotations) {
+            val annotationClass = annotation.annotationClass
+            val grouped = annotationsByClass.remove(annotationClass) ?: continue
+            if (grouped.size < 2) {
+                result.add(grouped.single())
+                continue
+            }
+
+            val containerClass = getOrCreateContainerClass(annotationClass)
+            if (containerClass != null)
+                wrapAnnotationEntriesInContainer(annotationClass, containerClass, grouped)?.let {
+                    result.add(it)
+                }
+            else logger.warnElement("Failed to find an annotation container class", annotationClass)
+        }
+        return result
+    }
+
+    // Adapted from RepeatedAnnotationLowering.kt
+    private fun getOrCreateContainerClass(annotationClass: IrClass): IrClass? {
+        val metaAnnotations = annotationClass.annotations
+        val jvmRepeatable =
+            metaAnnotations.find {
+                it.symbol.owner.parentAsClass.fqNameWhenAvailable ==
+                    JvmAnnotationNames.REPEATABLE_ANNOTATION
+            }
+        return if (jvmRepeatable != null) {
+            ((jvmRepeatable.getValueArgument(0) as? IrClassReference)?.symbol as? IrClassSymbol)
+                ?.owner
+        } else {
+            getOrCreateSyntheticRepeatableAnnotationContainer(annotationClass)
+        }
+    }
+
+    // Adapted from RepeatedAnnotationLowering.kt
+    private fun wrapAnnotationEntriesInContainer(
+        annotationClass: IrClass,
+        containerClass: IrClass,
+        entries: List<IrConstructorCall>
+    ): IrConstructorCall? {
+        val annotationType = annotationClass.typeWith()
+        val containerConstructor = containerClass.primaryConstructor
+        if (containerConstructor == null) {
+            logger.warnElement(
+                "Expected container class to have a primary constructor",
+                containerClass
+            )
+            return null
+        } else {
+            return IrConstructorCallImpl.fromSymbolOwner(
+                    containerClass.defaultType,
+                    containerConstructor.symbol
+                )
+                .apply {
+                    putValueArgument(
+                        0,
+                        IrVarargImpl(
+                            UNDEFINED_OFFSET,
+                            UNDEFINED_OFFSET,
+                            pluginContext.irBuiltIns.arrayClass.typeWith(annotationType),
+                            annotationType,
+                            entries
+                        )
+                    )
+                }
+        }
+    }
+
+    // Taken from AdditionalClassAnnotationLowering.kt
+    private fun getApplicableTargetSet(c: IrClass): Set<KotlinTarget>? {
+        val targetEntry = c.getAnnotation(StandardNames.FqNames.target) ?: return null
+        return loadAnnotationTargets(targetEntry)
+    }
+
+    // Taken from AdditionalClassAnnotationLowering.kt
+    private fun loadAnnotationTargets(targetEntry: IrConstructorCall): Set<KotlinTarget>? {
+        val valueArgument = targetEntry.getValueArgument(0) as? IrVararg ?: return null
+        return valueArgument.elements
+            .filterIsInstance<IrGetEnumValue>()
+            .mapNotNull { KotlinTarget.valueOrNull(it.symbol.owner.name.asString()) }
+            .toSet()
+    }
+
+    private val javaAnnotationTargetElementType by lazy {
+        extractor.referenceExternalClass("java.lang.annotation.ElementType")
+    }
+
+    private val javaAnnotationTarget by lazy {
+        extractor.referenceExternalClass("java.lang.annotation.Target")
+    }
+
+    private fun findEnumEntry(c: IrClass, name: String) =
+        c.declarations.filterIsInstance<IrEnumEntry>().find { it.name.asString() == name }
+
+    // Adapted from JvmSymbols.kt
+    private val jvm6TargetMap by lazy {
+        javaAnnotationTargetElementType?.let {
+            val etMethod = findEnumEntry(it, "METHOD")
+            mapOf(
+                KotlinTarget.CLASS to findEnumEntry(it, "TYPE"),
+                KotlinTarget.ANNOTATION_CLASS to findEnumEntry(it, "ANNOTATION_TYPE"),
+                KotlinTarget.CONSTRUCTOR to findEnumEntry(it, "CONSTRUCTOR"),
+                KotlinTarget.LOCAL_VARIABLE to findEnumEntry(it, "LOCAL_VARIABLE"),
+                KotlinTarget.FUNCTION to etMethod,
+                KotlinTarget.PROPERTY_GETTER to etMethod,
+                KotlinTarget.PROPERTY_SETTER to etMethod,
+                KotlinTarget.FIELD to findEnumEntry(it, "FIELD"),
+                KotlinTarget.VALUE_PARAMETER to findEnumEntry(it, "PARAMETER")
+            )
+        }
+    }
+
+    // Adapted from JvmSymbols.kt
+    private val jvm8TargetMap by lazy {
+        javaAnnotationTargetElementType?.let {
+            jvm6TargetMap?.let { j6Map ->
+                j6Map +
+                    mapOf(
+                        KotlinTarget.TYPE_PARAMETER to findEnumEntry(it, "TYPE_PARAMETER"),
+                        KotlinTarget.TYPE to findEnumEntry(it, "TYPE_USE")
+                    )
+            }
+        }
+    }
+
+    private fun getAnnotationTargetMap() =
+        if (pluginContext.platform?.any { it.targetPlatformVersion == JvmTarget.JVM_1_6 } == true)
+            jvm6TargetMap
+        else jvm8TargetMap
+
+    // Adapted from AdditionalClassAnnotationLowering.kt
+    private fun generateTargetAnnotation(c: IrClass): IrConstructorCall? {
+        if (c.hasAnnotation(JvmAnnotationNames.TARGET_ANNOTATION)) return null
+        val elementType = javaAnnotationTargetElementType ?: return null
+        val targetType = javaAnnotationTarget ?: return null
+        val targetConstructor =
+            targetType.declarations.firstIsInstanceOrNull<IrConstructor>() ?: return null
+        val targets = getApplicableTargetSet(c) ?: return null
+        val annotationTargetMap = getAnnotationTargetMap() ?: return null
+
+        val javaTargets =
+            targets
+                .mapNotNullTo(HashSet()) { annotationTargetMap[it] }
+                .sortedBy { ElementType.valueOf(it.symbol.owner.name.asString()) }
+        val vararg =
+            IrVarargImpl(
+                UNDEFINED_OFFSET,
+                UNDEFINED_OFFSET,
+                type = pluginContext.irBuiltIns.arrayClass.typeWith(elementType.defaultType),
+                varargElementType = elementType.defaultType
+            )
+        for (target in javaTargets) {
+            vararg.elements.add(
+                IrGetEnumValueImpl(
+                    UNDEFINED_OFFSET,
+                    UNDEFINED_OFFSET,
+                    elementType.defaultType,
+                    target.symbol
+                )
+            )
+        }
+
+        return IrConstructorCallImpl.fromSymbolOwner(
+                UNDEFINED_OFFSET,
+                UNDEFINED_OFFSET,
+                targetConstructor.returnType,
+                targetConstructor.symbol,
+                0
+            )
+            .apply { putValueArgument(0, vararg) }
+    }
+
+    private val javaAnnotationRetention by lazy {
+        extractor.referenceExternalClass("java.lang.annotation.Retention")
+    }
+    private val javaAnnotationRetentionPolicy by lazy {
+        extractor.referenceExternalClass("java.lang.annotation.RetentionPolicy")
+    }
+    private val javaAnnotationRetentionPolicyRuntime by lazy {
+        javaAnnotationRetentionPolicy?.let { findEnumEntry(it, "RUNTIME") }
+    }
+
+    private val annotationRetentionMap by lazy {
+        javaAnnotationRetentionPolicy?.let {
+            mapOf(
+                KotlinRetention.SOURCE to findEnumEntry(it, "SOURCE"),
+                KotlinRetention.BINARY to findEnumEntry(it, "CLASS"),
+                KotlinRetention.RUNTIME to javaAnnotationRetentionPolicyRuntime
+            )
+        }
+    }
+
+    // Taken from AnnotationCodegen.kt (not available in Kotlin < 1.6.20)
+    private fun IrClass.getAnnotationRetention(): KotlinRetention? {
+        val retentionArgument =
+            getAnnotation(StandardNames.FqNames.retention)?.getValueArgument(0) as? IrGetEnumValue
+                ?: return null
+        val retentionArgumentValue = retentionArgument.symbol.owner
+        return KotlinRetention.valueOf(retentionArgumentValue.name.asString())
+    }
+
+    // Taken from AdditionalClassAnnotationLowering.kt
+    private fun generateRetentionAnnotation(irClass: IrClass): IrConstructorCall? {
+        if (irClass.hasAnnotation(JvmAnnotationNames.RETENTION_ANNOTATION)) return null
+        val retentionMap = annotationRetentionMap ?: return null
+        val kotlinRetentionPolicy = irClass.getAnnotationRetention()
+        val javaRetentionPolicy =
+            kotlinRetentionPolicy?.let { retentionMap[it] }
+                ?: javaAnnotationRetentionPolicyRuntime
+                ?: return null
+        val retentionPolicyType = javaAnnotationRetentionPolicy ?: return null
+        val retentionType = javaAnnotationRetention ?: return null
+        val targetConstructor =
+            retentionType.declarations.firstIsInstanceOrNull<IrConstructor>() ?: return null
+
+        return IrConstructorCallImpl.fromSymbolOwner(
+                UNDEFINED_OFFSET,
+                UNDEFINED_OFFSET,
+                targetConstructor.returnType,
+                targetConstructor.symbol,
+                0
+            )
+            .apply {
+                putValueArgument(
+                    0,
+                    IrGetEnumValueImpl(
+                        UNDEFINED_OFFSET,
+                        UNDEFINED_OFFSET,
+                        retentionPolicyType.defaultType,
+                        javaRetentionPolicy.symbol
+                    )
+                )
+            }
+    }
+
+    private val javaAnnotationRepeatable by lazy {
+        extractor.referenceExternalClass("java.lang.annotation.Repeatable")
+    }
+    private val kotlinAnnotationRepeatableContainer by lazy {
+        extractor.referenceExternalClass("kotlin.jvm.internal.RepeatableContainer")
+    }
+
+    // Taken from declarationBuilders.kt (not available in Kotlin < 1.6):
+    private fun addDefaultGetter(p: IrProperty, parentClass: IrClass) {
+        val field =
+            p.backingField
+                ?: run {
+                    logger.warnElement("Expected property to have a backing field", p)
+                    return
+                }
+        p.addGetter {
+                origin = IrDeclarationOrigin.DEFAULT_PROPERTY_ACCESSOR
+                returnType = field.type
+            }
+            .apply {
+                val thisReceiever =
+                    parentClass.thisReceiver
+                        ?: run {
+                            logger.warnElement(
+                                "Expected property's parent class to have a receiver parameter",
+                                parentClass
+                            )
+                            return
+                        }
+                val newParam = copyParameterToFunction(thisReceiever, this)
+                dispatchReceiverParameter = newParam
+                body =
+                    factory
+                        .createBlockBody(UNDEFINED_OFFSET, UNDEFINED_OFFSET)
+                        .apply({
+                            this.statements.add(
+                                IrReturnImpl(
+                                    UNDEFINED_OFFSET,
+                                    UNDEFINED_OFFSET,
+                                    pluginContext.irBuiltIns.nothingType,
+                                    symbol,
+                                    IrGetFieldImpl(
+                                        UNDEFINED_OFFSET,
+                                        UNDEFINED_OFFSET,
+                                        field.symbol,
+                                        field.type,
+                                        IrGetValueImpl(
+                                            UNDEFINED_OFFSET,
+                                            UNDEFINED_OFFSET,
+                                            newParam.type,
+                                            newParam.symbol
+                                        )
+                                    )
+                                )
+                            )
+                        })
+            }
+    }
+
+    // Taken from JvmCachedDeclarations.kt
+    private fun getOrCreateSyntheticRepeatableAnnotationContainer(annotationClass: IrClass) =
+        extractor.globalExtensionState.syntheticRepeatableAnnotationContainers.getOrPut(
+            annotationClass
+        ) {
+            val containerClass =
+                pluginContext.irFactory
+                    .buildClass {
+                        kind = ClassKind.ANNOTATION_CLASS
+                        name = Name.identifier("Container")
+                    }
+                    .apply {
+                        createImplicitParameterDeclarationWithWrappedDescriptor()
+                        parent = annotationClass
+                        superTypes = listOf(getAnnotationType(pluginContext))
+                    }
+
+            val propertyName = Name.identifier("value")
+            val propertyType =
+                pluginContext.irBuiltIns.arrayClass.typeWith(annotationClass.typeWith())
+
+            containerClass
+                .addConstructor { isPrimary = true }
+                .apply { addValueParameter(propertyName.identifier, propertyType) }
+
+            containerClass
+                .addProperty { name = propertyName }
+                .apply property@{
+                    backingField =
+                        pluginContext.irFactory
+                            .buildField {
+                                name = propertyName
+                                type = propertyType
+                            }
+                            .apply {
+                                parent = containerClass
+                                correspondingPropertySymbol = this@property.symbol
+                            }
+                    addDefaultGetter(this, containerClass)
+                }
+
+            val repeatableContainerAnnotation =
+                kotlinAnnotationRepeatableContainer?.constructors?.single()
+
+            containerClass.annotations =
+                annotationClass.annotations
+                    .filter {
+                        it.isAnnotationWithEqualFqName(StandardNames.FqNames.retention) ||
+                            it.isAnnotationWithEqualFqName(StandardNames.FqNames.target)
+                    }
+                    .map { it.deepCopyWithSymbols(containerClass) } +
+                    listOfNotNull(
+                        repeatableContainerAnnotation?.let {
+                            IrConstructorCallImpl.fromSymbolOwner(
+                                UNDEFINED_OFFSET,
+                                UNDEFINED_OFFSET,
+                                it.returnType,
+                                it.symbol,
+                                0
+                            )
+                        }
+                    )
+
+            containerClass
+        }
+
+    // Adapted from AdditionalClassAnnotationLowering.kt
+    private fun generateRepeatableAnnotation(
+        irClass: IrClass,
+        extractAnnotationTypeAccesses: Boolean
+    ): IrConstructorCall? {
+        if (
+            !irClass.hasAnnotation(StandardNames.FqNames.repeatable) ||
+                irClass.hasAnnotation(JvmAnnotationNames.REPEATABLE_ANNOTATION)
+        )
+            return null
+
+        val repeatableConstructor =
+            javaAnnotationRepeatable?.declarations?.firstIsInstanceOrNull<IrConstructor>()
+                ?: return null
+
+        val containerClass = getOrCreateSyntheticRepeatableAnnotationContainer(irClass)
+        // Whenever a repeatable annotation with a Kotlin-synthesised container is extracted,
+        // extract the synthetic container to the same trap file.
+        extractor.extractClassSource(
+            containerClass,
+            extractDeclarations = true,
+            extractStaticInitializer = true,
+            extractPrivateMembers = true,
+            extractFunctionBodies = extractAnnotationTypeAccesses
+        )
+
+        val containerReference =
+            IrClassReferenceImpl(
+                UNDEFINED_OFFSET,
+                UNDEFINED_OFFSET,
+                pluginContext.irBuiltIns.kClassClass.typeWith(containerClass.defaultType),
+                containerClass.symbol,
+                containerClass.defaultType
+            )
+        return IrConstructorCallImpl.fromSymbolOwner(
+                UNDEFINED_OFFSET,
+                UNDEFINED_OFFSET,
+                repeatableConstructor.returnType,
+                repeatableConstructor.symbol,
+                0
+            )
+            .apply { putValueArgument(0, containerReference) }
+    }
+
+    private val javaAnnotationDocumented by lazy {
+        extractor.referenceExternalClass("java.lang.annotation.Documented")
+    }
+
+    // Taken from AdditionalClassAnnotationLowering.kt
+    private fun generateDocumentedAnnotation(irClass: IrClass): IrConstructorCall? {
+        if (
+            !irClass.hasAnnotation(StandardNames.FqNames.mustBeDocumented) ||
+                irClass.hasAnnotation(JvmAnnotationNames.DOCUMENTED_ANNOTATION)
+        )
+            return null
+
+        val documentedConstructor =
+            javaAnnotationDocumented?.declarations?.firstIsInstanceOrNull<IrConstructor>()
+                ?: return null
+
+        return IrConstructorCallImpl.fromSymbolOwner(
+            UNDEFINED_OFFSET,
+            UNDEFINED_OFFSET,
+            documentedConstructor.returnType,
+            documentedConstructor.symbol,
+            0
+        )
+    }
+
+    fun generateJavaMetaAnnotations(c: IrClass, extractAnnotationTypeAccesses: Boolean) =
+        // This is essentially AdditionalClassAnnotationLowering adapted to run outside the backend.
+        listOfNotNull(
+            generateTargetAnnotation(c),
+            generateRetentionAnnotation(c),
+            generateRepeatableAnnotation(c, extractAnnotationTypeAccesses),
+            generateDocumentedAnnotation(c)
+        )
+}
+*/

java/kotlin-extractor2/src/main/kotlin/PrimitiveTypeInfo.kt

@@ -0,0 +1,108 @@
+package com.github.codeql
+
+/*
+OLD: KE1
+import com.github.codeql.utils.*
+import com.github.codeql.utils.versions.*
+import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
+import org.jetbrains.kotlin.builtins.StandardNames
+import org.jetbrains.kotlin.ir.declarations.IrClass
+import org.jetbrains.kotlin.ir.declarations.IrPackageFragment
+import org.jetbrains.kotlin.ir.types.IrSimpleType
+import org.jetbrains.kotlin.ir.types.classOrNull
+
+class PrimitiveTypeMapping(val logger: Logger, val pluginContext: IrPluginContext) {
+    fun getPrimitiveInfo(s: IrSimpleType) =
+        s.classOrNull?.let {
+            if (
+                (it.owner.parent as? IrPackageFragment)?.packageFqName ==
+                    StandardNames.BUILT_INS_PACKAGE_FQ_NAME
+            )
+                mapping[it.owner.name]
+            else null
+        }
+
+    data class PrimitiveTypeInfo(
+        val primitiveName: String?,
+        val otherIsPrimitive: Boolean,
+        val javaClass: IrClass,
+        val kotlinPackageName: String,
+        val kotlinClassName: String
+    )
+
+    private fun findClass(fqName: String, fallback: IrClass): IrClass {
+        val symbol = getClassByFqName(pluginContext, fqName)
+        if (symbol == null) {
+            logger.warn("Can't find $fqName")
+            // Do the best we can
+            return fallback
+        } else {
+            return symbol.owner
+        }
+    }
+
+    private val mapping =
+        {
+            val kotlinByte = pluginContext.irBuiltIns.byteClass.owner
+            val javaLangByte = findClass("java.lang.Byte", kotlinByte)
+            val kotlinShort = pluginContext.irBuiltIns.shortClass.owner
+            val javaLangShort = findClass("java.lang.Short", kotlinShort)
+            val kotlinInt = pluginContext.irBuiltIns.intClass.owner
+            val javaLangInteger = findClass("java.lang.Integer", kotlinInt)
+            val kotlinLong = pluginContext.irBuiltIns.longClass.owner
+            val javaLangLong = findClass("java.lang.Long", kotlinLong)
+
+            val kotlinUByte = findClass("kotlin.UByte", kotlinByte)
+            val kotlinUShort = findClass("kotlin.UShort", kotlinShort)
+            val kotlinUInt = findClass("kotlin.UInt", kotlinInt)
+            val kotlinULong = findClass("kotlin.ULong", kotlinLong)
+
+            val kotlinDouble = pluginContext.irBuiltIns.doubleClass.owner
+            val javaLangDouble = findClass("java.lang.Double", kotlinDouble)
+            val kotlinFloat = pluginContext.irBuiltIns.floatClass.owner
+            val javaLangFloat = findClass("java.lang.Float", kotlinFloat)
+
+            val kotlinBoolean = pluginContext.irBuiltIns.booleanClass.owner
+            val javaLangBoolean = findClass("java.lang.Boolean", kotlinBoolean)
+
+            val kotlinChar = pluginContext.irBuiltIns.charClass.owner
+            val javaLangCharacter = findClass("java.lang.Character", kotlinChar)
+
+            val kotlinUnit = pluginContext.irBuiltIns.unitClass.owner
+
+            val kotlinNothing = pluginContext.irBuiltIns.nothingClass.owner
+            val javaLangVoid = findClass("java.lang.Void", kotlinNothing)
+
+            mapOf(
+                StandardNames.FqNames._byte.shortName() to
+                    PrimitiveTypeInfo("byte", true, javaLangByte, "kotlin", "Byte"),
+                StandardNames.FqNames._short.shortName() to
+                    PrimitiveTypeInfo("short", true, javaLangShort, "kotlin", "Short"),
+                StandardNames.FqNames._int.shortName() to
+                    PrimitiveTypeInfo("int", true, javaLangInteger, "kotlin", "Int"),
+                StandardNames.FqNames._long.shortName() to
+                    PrimitiveTypeInfo("long", true, javaLangLong, "kotlin", "Long"),
+                StandardNames.FqNames.uByteFqName.shortName() to
+                    PrimitiveTypeInfo("byte", true, kotlinUByte, "kotlin", "UByte"),
+                StandardNames.FqNames.uShortFqName.shortName() to
+                    PrimitiveTypeInfo("short", true, kotlinUShort, "kotlin", "UShort"),
+                StandardNames.FqNames.uIntFqName.shortName() to
+                    PrimitiveTypeInfo("int", true, kotlinUInt, "kotlin", "UInt"),
+                StandardNames.FqNames.uLongFqName.shortName() to
+                    PrimitiveTypeInfo("long", true, kotlinULong, "kotlin", "ULong"),
+                StandardNames.FqNames._double.shortName() to
+                    PrimitiveTypeInfo("double", true, javaLangDouble, "kotlin", "Double"),
+                StandardNames.FqNames._float.shortName() to
+                    PrimitiveTypeInfo("float", true, javaLangFloat, "kotlin", "Float"),
+                StandardNames.FqNames._boolean.shortName() to
+                    PrimitiveTypeInfo("boolean", true, javaLangBoolean, "kotlin", "Boolean"),
+                StandardNames.FqNames._char.shortName() to
+                    PrimitiveTypeInfo("char", true, javaLangCharacter, "kotlin", "Char"),
+                StandardNames.FqNames.unit.shortName() to
+                    PrimitiveTypeInfo("void", false, kotlinUnit, "kotlin", "Unit"),
+                StandardNames.FqNames.nothing.shortName() to
+                    PrimitiveTypeInfo(null, true, javaLangVoid, "kotlin", "Nothing"),
+            )
+        }()
+}
+*/

java/kotlin-extractor2/src/main/kotlin/TrapFileWriter.kt

@@ -0,0 +1,191 @@
+package com.github.codeql
+
+import java.io.BufferedInputStream
+import java.io.BufferedOutputStream
+import java.io.BufferedReader
+import java.io.BufferedWriter
+import java.io.File
+import java.io.FileInputStream
+import java.io.FileOutputStream
+import java.io.InputStreamReader
+import java.io.OutputStreamWriter
+import java.util.zip.GZIPInputStream
+import java.util.zip.GZIPOutputStream
+
+enum class Compression(val extension: String) {
+    NONE("") {
+        override fun bufferedWriter(file: File): BufferedWriter {
+            return file.bufferedWriter()
+        }
+    },
+    GZIP(".gz") {
+        override fun bufferedWriter(file: File): BufferedWriter {
+            return GZIPOutputStream(file.outputStream()).bufferedWriter()
+        }
+    };
+
+    abstract fun bufferedWriter(file: File): BufferedWriter
+}
+
+fun getCompression(logger: Logger): Compression {
+    val compression_env_var = "CODEQL_EXTRACTOR_JAVA_OPTION_TRAP_COMPRESSION"
+    val compression_option = System.getenv(compression_env_var)
+    val defaultCompression = Compression.GZIP
+    if (compression_option == null) {
+        return defaultCompression
+    } else {
+        try {
+            val compression_option_upper = compression_option.uppercase()
+            if (compression_option_upper == "BROTLI") {
+                logger.warn(
+                    "Kotlin extractor doesn't support Brotli compression. Using GZip instead."
+                )
+                return Compression.GZIP
+            } else {
+                return Compression.valueOf(compression_option_upper)
+            }
+        } catch (e: IllegalArgumentException) {
+            logger.warn(
+                "Unsupported compression type (\$$compression_env_var) \"$compression_option\". Supported values are ${
+                    Compression.values().joinToString()
+                }."
+            )
+            return defaultCompression
+        }
+    }
+}
+
+fun getTrapFileWriter(
+    compression: Compression,
+    logger: FileLogger,
+    checkTrapIdentical: Boolean,
+    trapFileName: String
+): TrapFileWriter {
+    return when (compression) {
+        Compression.NONE -> NonCompressedTrapFileWriter(logger, checkTrapIdentical, trapFileName)
+        Compression.GZIP -> GZipCompressedTrapFileWriter(logger, checkTrapIdentical, trapFileName)
+    }
+}
+
+abstract class TrapFileWriter(
+    val logger: FileLogger,
+    val checkTrapIdentical: Boolean,
+    trapName: String,
+    val extension: String
+) {
+    abstract protected fun getReader(file: File): BufferedReader
+    abstract protected fun getWriter(file: File): BufferedWriter
+
+    private val realFile = File(trapName + extension)
+    private val parentDir = realFile.parentFile
+
+    fun run(block: (bw: BufferedWriter) -> Unit) {
+        if (checkTrapIdentical || !exists()) {
+            parentDir.mkdirs()
+
+            logger.info("Will write TRAP file $realFile")
+            val tempFile = File.createTempFile(realFile.getName() + ".", ".trap.tmp" + extension, parentDir)
+            try {
+                logger.debug("Writing temporary TRAP file $tempFile")
+                getWriter(tempFile).use { bw -> block(bw) }
+
+                if (checkTrapIdentical && exists()) {
+                    if (equivalentTrap(getReader(tempFile), getReader(realFile))) {
+                        deleteTemp(tempFile)
+                    } else {
+                        renameTempToDifferent(tempFile)
+                    }
+                } else {
+                    renameTempToReal(tempFile)
+                }
+                // We catch Throwable rather than Exception, as we want to
+                // continue trying to extract everything else even if we get a
+                // stack overflow or an assertion failure in one file.
+            } catch (e: Throwable) {
+                logger.error("Extraction failed while writing '$tempFile'.", e)
+                /*
+                OLD: KE1
+                                fileExtractionProblems.setNonRecoverableProblem()
+                */
+            }
+        }
+    }
+
+    private fun exists(): Boolean {
+        return realFile.exists()
+    }
+
+    private fun deleteTemp(tempFile: File) {
+        if (!tempFile.delete()) {
+            logger.warn("Failed to delete $tempFile")
+        }
+    }
+
+    private fun renameTempToDifferent(tempFile: File) {
+        val trapDifferentFile =
+            File.createTempFile(realFile.getName() + ".", ".trap.different" + extension, parentDir)
+        if (tempFile.renameTo(trapDifferentFile)) {
+            logger.warn("TRAP difference: $realFile vs $trapDifferentFile")
+        } else {
+            logger.warn("Failed to rename $tempFile to $realFile")
+        }
+    }
+
+    private fun renameTempToReal(tempFile: File) {
+        if (!tempFile.renameTo(realFile)) {
+            logger.warn("Failed to rename $tempFile to $realFile")
+        }
+        logger.info("Finished writing TRAP file $realFile")
+    }
+}
+
+private class NonCompressedTrapFileWriter(logger: FileLogger, checkTrapIdentical: Boolean, trapName: String) :
+    TrapFileWriter(logger, checkTrapIdentical, trapName, "") {
+    override protected fun getReader(file: File): BufferedReader {
+        return file.bufferedReader()
+    }
+
+    override protected fun getWriter(file: File): BufferedWriter {
+        return file.bufferedWriter()
+    }
+}
+
+private class GZipCompressedTrapFileWriter(logger: FileLogger, checkTrapIdentical: Boolean, trapName: String) :
+    TrapFileWriter(logger, checkTrapIdentical, trapName, ".gz") {
+    override protected fun getReader(file: File): BufferedReader {
+        return BufferedReader(
+            InputStreamReader(GZIPInputStream(BufferedInputStream(FileInputStream(file))))
+        )
+    }
+
+    override protected fun getWriter(file: File): BufferedWriter {
+        return BufferedWriter(
+            OutputStreamWriter(GZIPOutputStream(BufferedOutputStream(FileOutputStream(file))))
+        )
+    }
+}
+
+/*
+This function determines whether 2 TRAP files should be considered to be
+equivalent. It returns `true` iff all of their non-comment lines are
+identical.
+*/
+private fun equivalentTrap(r1: BufferedReader, r2: BufferedReader): Boolean {
+    r1.use { br1 ->
+        r2.use { br2 ->
+            while (true) {
+                val l1 = br1.readLine()
+                val l2 = br2.readLine()
+                if (l1 == null && l2 == null) {
+                    return true
+                } else if (l1 == null || l2 == null) {
+                    return false
+                } else if (l1 != l2) {
+                    if (!l1.startsWith("//") || !l2.startsWith("//")) {
+                        return false
+                    }
+                }
+            }
+        }
+    }
+}

java/kotlin-extractor2/src/main/kotlin/TrapWriter.kt

@@ -0,0 +1,589 @@
+package com.github.codeql
+
+import LocallyVisibleFunctionLabels
+import com.intellij.openapi.util.TextRange
+import com.intellij.psi.PsiElement
+import com.intellij.psi.PsiFile
+import com.semmle.extractor.java.PopulateFile
+import com.semmle.util.unicode.UTF8Util
+import org.jetbrains.kotlin.analysis.api.symbols.KaFunctionSymbol
+import org.jetbrains.kotlin.analysis.api.symbols.KaVariableSymbol
+import java.io.BufferedWriter
+import java.io.File
+import java.util.concurrent.locks.ReentrantLock
+import kotlin.concurrent.withLock
+/*
+OLD: KE1
+import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
+import org.jetbrains.kotlin.ir.declarations.IrClass
+import org.jetbrains.kotlin.ir.declarations.IrFile
+import org.jetbrains.kotlin.ir.declarations.IrFunction
+import org.jetbrains.kotlin.ir.declarations.IrVariable
+import org.jetbrains.kotlin.ir.declarations.path
+import org.jetbrains.kotlin.ir.expressions.IrCall
+import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET
+*/
+import org.jetbrains.kotlin.psi.*
+
+/**
+ * Each `.trap` file has a `TrapLabelManager` while we are writing it. It provides fresh TRAP label
+ * names, and maintains a mapping from keys (`@"..."`) to labels.
+ */
+class TrapLabelManager {
+    /**
+     * The lock that controls access to the label manager state.
+     * While we can make a thread-safe MutableMap with
+     * Collections.synchronizedMap and use getOrPut, that doesn't
+     * guarantee not to run the `defaultValue` function when it isn't
+     * necessary, which makes it useless for our purposes.
+     * TODO: We only actually need this for the diagnostic TRAP file. Make it optional?
+     */
+    private val lock = ReentrantLock()
+
+    /** The next integer to use as a label name. */
+    private var nextInt: Int = 100
+
+    /** Returns a fresh label. */
+    fun <T : AnyDbType> getFreshLabel(): Label<T> {
+        lock.withLock {
+            return IntLabel(nextInt++)
+        }
+    }
+
+    /** A mapping from a key (`@"..."`) to the label defined to be that key, if any. */
+    private val labelMapping: MutableMap<String, Label<*>> = mutableMapOf<String, Label<*>>()
+
+    fun <T> withLabelMapping(action: (MutableMap<String, Label<*>>) -> T): T {
+        lock.withLock {
+            return action(labelMapping)
+        }
+    }
+
+    // TODO: This will only be used in a TrapWriter for a source file, not a diagnostic TRAP writer. Consider moving this out of this class
+    private val locallyVisibleFunctionLabelMapping: MutableMap<KaFunctionSymbol, LocallyVisibleFunctionLabels> =
+        mutableMapOf()
+
+    fun getLocallyVisibleFunctionLabelMapping(
+        key: KaFunctionSymbol
+    ): LocallyVisibleFunctionLabels {
+        return locallyVisibleFunctionLabelMapping[key]!!
+    }
+
+    fun getOrAddLocallyVisibleFunctionLabelMapping(
+        key: KaFunctionSymbol,
+        add: (KaFunctionSymbol) -> LocallyVisibleFunctionLabels
+    ): LocallyVisibleFunctionLabels {
+        val res = locallyVisibleFunctionLabelMapping[key]
+        if (res != null) {
+            return res
+        }
+
+        val labels = add(key)
+        locallyVisibleFunctionLabelMapping[key] = labels
+        return labels
+    }
+
+    /*
+    OLD: KE1
+        val anonymousTypeMapping: MutableMap<IrClass, TypeResults> = mutableMapOf()
+
+        /**
+         * The set of labels of generic specialisations that we have extracted in this TRAP file. We
+         * can't easily avoid duplication between TRAP files, as the labels contain references to other
+         * labels, so we just accept this duplication.
+         */
+        val genericSpecialisationsExtracted = HashSet<String>()
+    */
+
+    /**
+     * Sometimes, when we extract a file class we don't have the KtFile for it, so we are not able
+     * to give it a location. This means that the location is written outside of the label creation.
+     * This allows us to keep track of whether we've written the location already in this TRAP file,
+     * to avoid duplication.
+     */
+    private val fileClassLocationsExtracted = HashSet<KtFile>()
+
+    /**
+     * Indicate that we want `file`'s file class location marked as extracted.
+     * Returns true if we need to actually write the TRAP for it, or false
+     * if it's already been done.
+     */
+    fun markFileClassLocationAsExtracted(file: KtFile): Boolean {
+        lock.withLock {
+            return fileClassLocationsExtracted.add(file)
+        }
+    }
+}
+
+/**
+ * A `TrapWriter` is used to write TRAP to a particular TRAP file. There may be multiple
+ * `TrapWriter`s for the same file, as different instances will have different additional state, but
+ * they must all share the same `TrapLabelManager` and `BufferedWriter`.
+ * `BasicLogger`s, `TrapLabelManager` and `BufferedWriter` are threadsafe, so `TrapWriter`s are too.
+ */
+abstract class TrapWriter(
+    protected val basicLogger: BasicLogger,
+    val lm: TrapLabelManager,
+    private val bw: BufferedWriter
+) {
+    abstract fun getDiagnosticTrapWriter(): DiagnosticTrapWriter
+
+    /*
+    OLD: KE1
+        /**
+         * Returns the label that is defined to be the given key, if such a label exists, and `null`
+         * otherwise. Most users will want to use `getLabelFor` instead, which allows non-existent
+         * labels to be initialised.
+         */
+    TODO: Inline this if it can remain private
+    */
+    private fun <T : AnyDbType> getExistingLabelFor(key: String): Label<T>? {
+        return lm.withLabelMapping { labelMapping -> labelMapping.get(key)?.cast<T>() }
+    }
+
+    /**
+     * Returns the label for the given key, if one exists. Otherwise, a fresh label is bound to that
+     * key, `initialise` is run on it, and it is returned.
+     */
+    @JvmOverloads // Needed so Java can call a method with an optional argument
+    fun <T : AnyDbType> getLabelFor(key: String, initialise: (Label<T>) -> Unit = {}): Label<T> {
+        return lm.withLabelMapping { labelMapping ->
+            val maybeLabel: Label<T>? = getExistingLabelFor(key)
+            if (maybeLabel == null) {
+                val label: Label<T> = lm.getFreshLabel()
+                labelMapping.put(key, label)
+                writeTrap("$label = $key\n")
+                initialise(label)
+                label
+            } else {
+                maybeLabel
+            }
+        }
+    }
+
+    /** Returns a label for a fresh ID (i.e. a new label bound to `*`). */
+    fun <T : AnyDbType> getFreshIdLabel(): Label<T> {
+        val label: Label<T> = lm.getFreshLabel()
+        writeTrap("$label = *\n")
+        return label
+    }
+
+    /**
+     * It is not easy to assign keys to local variables, so they get given `*` IDs. However, the
+     * same variable may be referred to from distant places in the IR, so we need a way to find out
+     * which label is used for a given local variable. This information is stored in this mapping.
+     */
+    // TODO: This should be in a subclass so that DiagnosticTrapWriter doesn't include it, as it is not threadsafe
+    private val variableLabelMapping: MutableMap<KaVariableSymbol, Label<out DbLocalvar>> =
+        mutableMapOf<KaVariableSymbol, Label<out DbLocalvar>>()
+
+    /** This returns the label used for a local variable, creating one if none currently exists. */
+    fun <T> getVariableLabelFor(v: KaVariableSymbol): Label<out DbLocalvar> {
+        val maybeLabel = variableLabelMapping[v]
+        if (maybeLabel == null) {
+            val label = getFreshIdLabel<DbLocalvar>()
+            variableLabelMapping.put(v, label)
+            return label
+        } else {
+            return maybeLabel
+        }
+    }
+
+    /*
+    OLD: KE1
+    fun getExistingVariableLabelFor(v: KtProperty): Label<out DbLocalvar>? {
+        return variableLabelMapping[v]
+    }
+    */
+
+    /**
+     * This returns a label for the location described by its arguments. Typically users will not
+     * want to call this directly, but instead use `unknownLocation`, or overloads of this defined
+     * by subclasses.
+     */
+    fun getLocation(
+        fileId: Label<DbFile>,
+        startLine: Int,
+        startColumn: Int,
+        endLine: Int,
+        endColumn: Int
+    ): Label<DbLocation> {
+        return getLabelFor("@\"loc,{$fileId},$startLine,$startColumn,$endLine,$endColumn\"") {
+            writeLocations_default(it, fileId, startLine, startColumn, endLine, endColumn)
+        }
+    }
+
+    /**
+     * The label for the 'unknown' location. This is lazy, as we don't want to define it in a TRAP
+     * file unless the TRAP file actually contains something with an 'unknown' location.
+     */
+    val unknownLocation: Label<DbLocation> by lazy {
+        val unknownFileLabel = "@\";sourcefile\""
+        val unknownFileId = getLabelFor(unknownFileLabel, { writeFiles(it, "") })
+        getWholeFileLocation(unknownFileId)
+    }
+
+    /**
+     * Returns the label for the file `filePath`. If `populateFileTables` is true, then this also
+     * adds rows to the `files` and `folders` tables for this file.
+     */
+    fun mkFileId(filePath: String, populateFileTables: Boolean): Label<DbFile> {
+        // If a file is in a jar, then the Kotlin compiler gives
+        // `<jar file>!/<path within jar>` as its path. We need to split
+        // it as appropriate, to make the right file ID.
+        val populateFile = PopulateFile(this)
+        val splitFilePath = filePath.split("!/")
+        if (splitFilePath.size == 1) {
+            return populateFile.getFileLabel(File(filePath), populateFileTables)
+        } else {
+            return populateFile.getFileInJarLabel(
+                File(splitFilePath.get(0)),
+                splitFilePath.get(1),
+                populateFileTables
+            )
+        }
+    }
+
+    /**
+     * If you have an ID for a file, then this gets a label for the location representing the whole
+     * of that file.
+     */
+    fun getWholeFileLocation(fileId: Label<DbFile>): Label<DbLocation> {
+        return getLocation(fileId, 0, 0, 0, 0)
+    }
+
+    /**
+     * Write a raw string into the TRAP file.
+     * The only external caller of this should be the generated
+     * KotlinExtractorDbScheme.kt.
+     */
+    fun writeTrap(trap: String) {
+        bw.write(trap)
+    }
+
+    /** Write a comment into the TRAP file. */
+    fun writeComment(comment: String) {
+        writeTrap("// ${comment.replace("\n", "\n//    ")}\n")
+    }
+
+    /** Flush the TRAP file. */
+    fun flush() {
+        bw.flush()
+    }
+
+    /**
+     * Escape a string so that it can be used in a TRAP string literal, i.e. with `"` escaped as
+     * `""`.
+     * The only external caller of this should be the generated
+     * KotlinExtractorDbScheme.kt.
+     */
+    fun escapeTrapString(str: String) = str.replace("\"", "\"\"")
+
+    /** TRAP string literals are limited to 1 megabyte. */
+    private val MAX_STRLEN = 1.shl(20)
+
+    /**
+     * Truncate a string, if necessary, so that it can be used as a TRAP string literal. TRAP string
+     * literals are limited to 1 megabyte.
+     * The only external caller of this should be the generated
+     * KotlinExtractorDbScheme.kt.
+     */
+    fun truncateString(str: String): String {
+        val len = str.length
+        val newLen = UTF8Util.encodablePrefixLength(str, MAX_STRLEN)
+        if (newLen < len) {
+            basicLogger.warn(
+                this.getDiagnosticTrapWriter(),
+                "Truncated string of length $len",
+                "Truncated string of length $len, starting '${str.take(100)}', ending '${str.takeLast(100)}'"
+            )
+            return str.take(newLen)
+        } else {
+            return str
+        }
+    }
+
+    /**
+     * Gets a FileTrapWriter like this one (using the same label manager, writer etc), but using the
+     * given `filePath` for locations.
+     */
+    fun makeFileTrapWriter(filePath: String, populateFileTables: Boolean) =
+        FileTrapWriter(
+            basicLogger,
+            lm,
+            bw,
+            this.getDiagnosticTrapWriter(),
+            filePath,
+            populateFileTables
+        )
+
+    /**
+     * Gets a FileTrapWriter like this one (using the same label manager, writer etc), but using the
+     * given `IrFile` for locations.
+     */
+    fun makeSourceFileTrapWriter(file: KtFile, populateFileTables: Boolean) =
+        SourceFileTrapWriter(
+            basicLogger,
+            lm,
+            bw,
+            this.getDiagnosticTrapWriter(),
+            file,
+            populateFileTables
+        )
+}
+
+/** A `PlainTrapWriter` has no additional context of its own. */
+class PlainTrapWriter(
+    basicLogger: BasicLogger,
+    lm: TrapLabelManager,
+    bw: BufferedWriter,
+    val dtw: DiagnosticTrapWriter
+) : TrapWriter(basicLogger, lm, bw) {
+    override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter {
+        return dtw
+    }
+}
+
+/**
+ * A `DiagnosticTrapWriter` is a TrapWriter that diagnostics can be written to; i.e. it has
+ * the #compilation label defined. In practice, this means that it is a TrapWriter for the
+ * invocation TRAP file.
+ */
+class DiagnosticTrapWriter(basicLogger: BasicLogger, lm: TrapLabelManager, bw: BufferedWriter) :
+    TrapWriter(basicLogger, lm, bw) {
+    override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter {
+        return this
+    }
+}
+
+/*
+OLD: KE1
+/**
+ * A `FileTrapWriter` is used when we know which file we are extracting entities from, so we can at
+ * least give the right file as a location.
+ *
+ * An ID for the file will be created, and if `populateFileTables` is true then we will also add
+ * rows to the `files` and `folders` tables for it.
+ */
+*/
+open class FileTrapWriter(
+    basicLogger: BasicLogger,
+    lm: TrapLabelManager,
+    bw: BufferedWriter,
+    val dtw: DiagnosticTrapWriter,
+    val filePath: String,
+    populateFileTables: Boolean
+) : TrapWriter(basicLogger, lm, bw) {
+
+    /** The ID for the file that we are extracting from. */
+    val fileId = mkFileId(filePath, populateFileTables)
+
+    override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter {
+        return dtw
+    }
+
+    /*
+    OLD: KE1
+        private fun offsetMinOf(default: Int, vararg options: Int?): Int {
+            if (default == UNDEFINED_OFFSET || default == SYNTHETIC_OFFSET) {
+                return default
+            }
+
+            var currentMin = default
+            for (option in options) {
+                if (
+                    option != null &&
+                        option != UNDEFINED_OFFSET &&
+                        option != SYNTHETIC_OFFSET &&
+                        option < currentMin
+                ) {
+                    currentMin = option
+                }
+            }
+
+            return currentMin
+        }
+
+        private fun getStartOffset(e: IrElement): Int {
+            return when (e) {
+                is IrCall -> {
+                    // Calls have incorrect startOffset, so we adjust them:
+                    val dr = e.dispatchReceiver?.let { getStartOffset(it) }
+                    val er = e.extensionReceiver?.let { getStartOffset(it) }
+                    offsetMinOf(e.startOffset, dr, er)
+                }
+                else -> e.startOffset
+            }
+        }
+
+        private fun getEndOffset(e: IrElement): Int {
+            return e.endOffset
+        }
+    */
+
+    private data class Location(val startLine: Int, val startColumn: Int, val endLine: Int, val endColumn: Int)
+
+    private fun getLocationInfo(file: PsiFile, range: TextRange): Location {
+        val document = file.getViewProvider().getDocument()
+        val start = range.getStartOffset()
+        val startLine0 = document.getLineNumber(start)
+        val startCol0 = start - document.getLineStartOffset(startLine0)
+        val end = range.getEndOffset()
+        val endLine0 = document.getLineNumber(end)
+        val endCol1 = end - document.getLineStartOffset(endLine0)
+        // TODO: unknown/synthetic locations?
+        return Location(startLine0 + 1, startCol0 + 1, endLine0 + 1, endCol1)
+    }
+
+    /** Gets a label for the location of `e`. */
+    fun getLocation(file: PsiFile, range: TextRange): Label<DbLocation> {
+        val loc = getLocationInfo(file, range)
+        return getLocation(fileId, loc.startLine, loc.startColumn, loc.endLine, loc.endColumn)
+    }
+
+    /** Gets a label for the location of `e`. */
+    fun getLocation(e: PsiElement): Label<DbLocation> {
+        val file = e.getContainingFile()
+        val range = e.getTextRange()
+        return getLocation(file, range)
+    }
+
+    /*
+    OLD: KE1
+        /** Gets a label for the location of `e`. */
+        fun getLocation(e: IrElement): Label<DbLocation> {
+            return getLocation(getStartOffset(e), getEndOffset(e))
+        }
+        /**
+         * Gets a label for the location corresponding to `startOffset` and `endOffset` within this
+         * file.
+         */
+        open fun getLocation(startOffset: Int, endOffset: Int): Label<DbLocation> {
+            // We don't have a FileEntry to look up the offsets in, so all
+            // we can do is return a whole-file location.
+            return getWholeFileLocation()
+        }
+    */
+
+    /**
+     * Gets the location of `e` as a human-readable string. Only used in log messages and exception
+     * messages.
+     */
+    /* TODO open */ fun getLocationString(e: PsiElement): String {
+        val file = e.getContainingFile()
+        val range = e.getTextRange()
+        val loc = getLocationInfo(file, range)
+        return "file://$filePath:${loc.startLine}:${loc.startColumn}:${loc.endLine}:${loc.endColumn}"
+
+        /*
+        OLD: KE1
+                // We don't have a FileEntry to look up the offsets in, so all
+                // we can do is return a whole-file location. We omit the
+                // `:0:0:0:0` so that it is easy to distinguish from a location
+                // where we have actually determined the start/end lines/columns
+                // to be 0.
+                return "file://$filePath"
+        */
+    }
+
+    /** Gets a label for the location representing the whole of this file. */
+    fun getWholeFileLocation(): Label<DbLocation> {
+        return getWholeFileLocation(fileId)
+    }
+}
+
+/*
+OLD: KE1
+/**
+ * A `SourceFileTrapWriter` is used when not only do we know which file we are extracting entities
+ * from, but we also have an `IrFileEntry` (from an `IrFile`) which allows us to map byte offsets to
+ * line and column numbers.
+ *
+ * An ID for the file will be created, and if `populateFileTables` is true then we will also add
+ * rows to the `files` and `folders` tables for it.
+ */
+*/
+class SourceFileTrapWriter(
+    basicLogger: BasicLogger,
+    lm: TrapLabelManager,
+    bw: BufferedWriter,
+    dtw: DiagnosticTrapWriter,
+    val ktFile: KtFile,
+    populateFileTables: Boolean
+) : FileTrapWriter(basicLogger, lm, bw, dtw, ktFile.virtualFilePath, populateFileTables) {
+    /*
+    OLD: KE1
+        /**
+         * The file entry for the file that we are extracting from. Used to map offsets to line/column
+         * numbers.
+         */
+        private val fileEntry = irFile.fileEntry
+
+        override fun getLocation(startOffset: Int, endOffset: Int): Label<DbLocation> {
+            if (startOffset == UNDEFINED_OFFSET || endOffset == UNDEFINED_OFFSET) {
+                if (startOffset != endOffset) {
+                    basicLogger.warn(
+                        dtw,
+                        "Location with inconsistent offsets (start $startOffset, end $endOffset)",
+                        null
+                    )
+                }
+                return getWholeFileLocation()
+            }
+
+            if (startOffset == SYNTHETIC_OFFSET || endOffset == SYNTHETIC_OFFSET) {
+                if (startOffset != endOffset) {
+                    basicLogger.warn(
+                        dtw,
+                        "Location with inconsistent offsets (start $startOffset, end $endOffset)",
+                        null
+                    )
+                }
+                return getWholeFileLocation()
+            }
+
+            // If this is the location for a compiler-generated element, then it will
+            // be a zero-width location. QL doesn't support these, so we translate it
+            // into a one-width location.
+            val endColumnOffset = if (startOffset == endOffset) 1 else 0
+            return getLocation(
+                fileId,
+                fileEntry.getLineNumber(startOffset) + 1,
+                fileEntry.getColumnNumber(startOffset) + 1,
+                fileEntry.getLineNumber(endOffset) + 1,
+                fileEntry.getColumnNumber(endOffset) + endColumnOffset
+            )
+        }
+
+        override fun getLocationString(e: IrElement): String {
+            if (e.startOffset == UNDEFINED_OFFSET || e.endOffset == UNDEFINED_OFFSET) {
+                if (e.startOffset != e.endOffset) {
+                    basicLogger.warn(
+                        dtw,
+                        "Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})",
+                        null
+                    )
+                }
+                return "<unknown location while processing $filePath>"
+            }
+
+            if (e.startOffset == SYNTHETIC_OFFSET || e.endOffset == SYNTHETIC_OFFSET) {
+                if (e.startOffset != e.endOffset) {
+                    basicLogger.warn(
+                        dtw,
+                        "Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})",
+                        null
+                    )
+                }
+                return "<synthetic location while processing $filePath>"
+            }
+
+            val startLine = fileEntry.getLineNumber(e.startOffset) + 1
+            val startColumn = fileEntry.getColumnNumber(e.startOffset) + 1
+            val endLine = fileEntry.getLineNumber(e.endOffset) + 1
+            val endColumn = fileEntry.getColumnNumber(e.endOffset)
+            return "file://$filePath:$startLine:$startColumn:$endLine:$endColumn"
+        }
+    */
+}

java/kotlin-extractor2/src/main/kotlin/comments/CommentExtractor.kt

@@ -0,0 +1,97 @@
+package com.github.codeql.comments
+
+/*
+OLD: KE1
+import com.github.codeql.*
+import com.github.codeql.utils.isLocalFunction
+import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.declarations.*
+import org.jetbrains.kotlin.ir.expressions.IrBody
+import org.jetbrains.kotlin.ir.expressions.IrExpression
+import org.jetbrains.kotlin.ir.util.parentClassOrNull
+
+open class CommentExtractor(
+    protected val fileExtractor: KotlinFileExtractor,
+    protected val file: IrFile,
+    protected val fileLabel: Label<out DbFile>
+) {
+    protected val tw = fileExtractor.tw
+    protected val logger = fileExtractor.logger
+
+    protected fun getLabel(element: IrElement): Label<out DbTop>? {
+        if (element == file) return fileLabel
+
+        if (element is IrValueParameter && element.index == -1) {
+            // Don't attribute comments to the implicit `this` parameter of a function.
+            return null
+        }
+
+        val label: String
+        val existingLabel =
+            if (element is IrVariable) {
+                // local variables are not named globally, so we need to get them from the variable
+                // label cache
+                label = "variable ${element.name.asString()}"
+                tw.getExistingVariableLabelFor(element)
+            } else if (element is IrFunction && element.isLocalFunction()) {
+                // local functions are not named globally, so we need to get them from the local
+                // function label cache
+                label = "local function ${element.name.asString()}"
+                fileExtractor.getExistingLocallyVisibleFunctionLabel(element)
+            } else {
+                label = getLabelForNamedElement(element) ?: return null
+                tw.getExistingLabelFor<DbTop>(label)
+            }
+        if (existingLabel == null) {
+            // Sometimes we don't extract elements.
+            if (element !is IrDeclarationWithVisibility) {
+                logger.warn("Couldn't get existing label for $label")
+            }
+            return null
+        }
+        return existingLabel
+    }
+
+    private fun getLabelForNamedElement(element: IrElement): String? {
+        when (element) {
+            is IrClass -> return fileExtractor.getClassLabel(element, listOf()).classLabel
+            is IrTypeParameter -> return fileExtractor.getTypeParameterLabel(element)
+            is IrFunction -> {
+                return if (element.isLocalFunction()) {
+                    null
+                } else {
+                    fileExtractor.getFunctionLabel(element, null)
+                }
+            }
+            is IrValueParameter -> return fileExtractor.getValueParameterLabel(element, null)
+            is IrProperty -> return fileExtractor.getPropertyLabel(element)
+            is IrField -> return fileExtractor.getFieldLabel(element)
+            is IrEnumEntry -> return fileExtractor.getEnumEntryLabel(element)
+            is IrTypeAlias -> return fileExtractor.getTypeAliasLabel(element)
+            is IrAnonymousInitializer -> {
+                val parentClass = element.parentClassOrNull
+                if (parentClass == null) {
+                    logger.warnElement("Parent of anonymous initializer is not a class", element)
+                    return null
+                }
+                // Assign the comment to the class. The content of the `init` blocks might be
+                // extracted in multiple constructors.
+                return getLabelForNamedElement(parentClass)
+            }
+
+            // Fresh entities, not named elements:
+            is IrBody -> return null
+            is IrExpression -> return null
+
+            // todo add others:
+            else -> {
+                logger.warnElement(
+                    "Unhandled element type found during comment extraction: ${element::class}",
+                    element
+                )
+                return null
+            }
+        }
+    }
+}
+*/

java/kotlin-extractor2/src/main/kotlin/comments/CommentExtractorPSI.kt

@@ -0,0 +1,125 @@
+package com.github.codeql.comments
+
+/*
+OLD: KE1
+import com.github.codeql.*
+import com.github.codeql.utils.IrVisitorLookup
+import com.github.codeql.utils.Psi2IrFacade
+import com.github.codeql.utils.versions.getPsi2Ir
+import com.intellij.psi.PsiComment
+import com.intellij.psi.PsiElement
+import org.jetbrains.kotlin.config.KotlinCompilerVersion
+import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.declarations.*
+import org.jetbrains.kotlin.kdoc.psi.api.KDoc
+import org.jetbrains.kotlin.lexer.KtTokens
+import org.jetbrains.kotlin.psi.KtVisitor
+import org.jetbrains.kotlin.psi.psiUtil.endOffset
+import org.jetbrains.kotlin.psi.psiUtil.startOffset
+
+class CommentExtractorPSI(
+    fileExtractor: KotlinFileExtractor,
+    file: IrFile,
+    fileLabel: Label<out DbFile>
+) : CommentExtractor(fileExtractor, file, fileLabel) {
+    // Returns true if it extracted the comments; false otherwise.
+    fun extract(): Boolean {
+        val psi2Ir = getPsi2Ir()
+        if (psi2Ir == null) {
+            logger.warn(
+                "Comments will not be extracted as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})"
+            )
+            return false
+        }
+        val ktFile = psi2Ir.getKtFile(file)
+        if (ktFile == null) {
+            return false
+        }
+        val commentVisitor = mkCommentVisitor(psi2Ir)
+        ktFile.accept(commentVisitor)
+        return true
+    }
+
+    private fun mkCommentVisitor(psi2Ir: Psi2IrFacade): KtVisitor<Unit, Unit> =
+        object : KtVisitor<Unit, Unit>() {
+            override fun visitElement(element: PsiElement) {
+                element.acceptChildren(this)
+
+                // Slightly hacky, but `visitComment` doesn't seem to visit comments with
+                // `tokenType` `KtTokens.DOC_COMMENT`
+                if (element is PsiComment) {
+                    visitCommentElement(element)
+                }
+            }
+
+            private fun visitCommentElement(comment: PsiComment) {
+                val type: CommentType =
+                    when (comment.tokenType) {
+                        KtTokens.EOL_COMMENT -> {
+                            CommentType.SingleLine
+                        }
+                        KtTokens.BLOCK_COMMENT -> {
+                            CommentType.Block
+                        }
+                        KtTokens.DOC_COMMENT -> {
+                            CommentType.Doc
+                        }
+                        else -> {
+                            logger.warn("Unhandled comment token type: ${comment.tokenType}")
+                            return
+                        }
+                    }
+
+                val commentLabel = tw.getFreshIdLabel<DbKtcomment>()
+                tw.writeKtComments(commentLabel, type.value, comment.text)
+                val locId = tw.getLocation(comment.startOffset, comment.endOffset)
+                tw.writeHasLocation(commentLabel, locId)
+
+                if (comment.tokenType != KtTokens.DOC_COMMENT) {
+                    return
+                }
+
+                if (comment !is KDoc) {
+                    logger.warn("Unexpected comment type with DocComment token type.")
+                    return
+                }
+
+                for (sec in comment.getAllSections()) {
+                    val commentSectionLabel = tw.getFreshIdLabel<DbKtcommentsection>()
+                    tw.writeKtCommentSections(commentSectionLabel, commentLabel, sec.getContent())
+                    val name = sec.name
+                    if (name != null) {
+                        tw.writeKtCommentSectionNames(commentSectionLabel, name)
+                    }
+                    val subjectName = sec.getSubjectName()
+                    if (subjectName != null) {
+                        tw.writeKtCommentSectionSubjectNames(commentSectionLabel, subjectName)
+                    }
+                }
+
+                // Only storing the owner of doc comments:
+                val ownerPsi = getKDocOwner(comment) ?: return
+
+                val owners = mutableListOf<IrElement>()
+                file.accept(IrVisitorLookup(psi2Ir, ownerPsi, file), owners)
+
+                for (ownerIr in owners) {
+                    val ownerLabel = getLabel(ownerIr)
+                    if (ownerLabel != null) {
+                        tw.writeKtCommentOwners(commentLabel, ownerLabel)
+                    }
+                }
+            }
+
+            private fun getKDocOwner(comment: KDoc): PsiElement? {
+                val owner = comment.owner
+                if (owner == null) {
+                    logger.warn(
+                        "Couldn't get owner of KDoc. The comment is extracted without an owner."
+                    )
+                }
+                return owner
+            }
+        }
+}
+*/

java/kotlin-extractor2/src/main/kotlin/comments/CommentType.kt

@@ -0,0 +1,10 @@
+package com.github.codeql.comments
+
+/*
+OLD: KE1
+enum class CommentType(val value: Int) {
+    SingleLine(1),
+    Block(2),
+    Doc(3)
+}
+*/

java/kotlin-extractor2/src/main/kotlin/entities/Class.kt

@@ -0,0 +1,385 @@
+package com.github.codeql
+
+import com.github.codeql.entities.extractTypeParameter
+import com.github.codeql.entities.getTypeArgumentLabel
+import com.github.codeql.utils.isInterfaceLike
+import org.jetbrains.kotlin.analysis.api.KaExperimentalApi
+import org.jetbrains.kotlin.analysis.api.KaSession
+import org.jetbrains.kotlin.analysis.api.symbols.*
+import org.jetbrains.kotlin.analysis.api.types.KaTypeProjection
+
+context(KaSession)
+@OptIn(KaExperimentalApi::class)
+fun KotlinFileExtractor.extractClassSource(
+    c: KaClassSymbol,
+    extractDeclarations: Boolean,
+    /*
+    OLD: KE1
+    extractStaticInitializer: Boolean,
+    extractPrivateMembers: Boolean,
+    extractFunctionBodies: Boolean
+     */
+): Label<out DbClassorinterface> {
+    with("class source", c) {
+        // OLD: KE1: DeclarationStackAdjuster(c).use {
+            val id = useClassSource(c)
+            val pkg = c.classId?.packageFqName?.asString() ?: ""
+            val cls =
+                if (c.classKind == KaClassKind.ANONYMOUS_OBJECT) "" else c.name!!.asString() // TODO: Remove !!
+            val pkgId = extractPackage(pkg)
+            tw.writeClasses_or_interfaces(id, cls, pkgId, id)
+            if (c.isInterfaceLike) {
+                tw.writeIsInterface(id)
+                if (c.classKind == KaClassKind.ANNOTATION_CLASS) {
+                    tw.writeIsAnnotType(id)
+                }
+            } else {
+                val kind = c.classKind
+                if (kind == KaClassKind.ENUM_CLASS) {
+                    tw.writeIsEnumType(id)
+                } else if (
+                    kind != KaClassKind.CLASS &&
+                    kind != KaClassKind.OBJECT //&&
+                //OLD KE1: kind != ClassKind.ENUM_ENTRY
+                ) else {
+                    logger.warnElement("Unrecognised class kind $kind", c)
+                }
+
+                /*
+                OLD: KE1
+                if (c.origin == IrDeclarationOrigin.FILE_CLASS) {
+                    tw.writeFile_class(id)
+                }
+                 */
+
+                if ((c as? KaNamedClassSymbol)?.isData == true) {
+                    tw.writeKtDataClasses(id)
+                }
+            }
+
+            val locId = PsiElementOrSymbol.of(c).getLocation(tw)
+            tw.writeHasLocation(id, locId)
+
+            // OLD: KE1
+            //extractEnclosingClass(c.parent, id, c, locId, listOf())
+            //val javaClass = (c.source as? JavaSourceElement)?.javaElement as? JavaClass
+
+            c.typeParameters.mapIndexed { idx, param ->
+                extractTypeParameter(param, idx, /* OLD: KE1 javaClass?.typeParameters?.getOrNull(idx) */)
+            }
+            if (extractDeclarations) {
+                if (c.classKind == KaClassKind.ANNOTATION_CLASS) {
+                    c.declaredMemberScope.declarations.filterIsInstance<KaPropertySymbol>().forEach {
+                        val getter = it.getter
+                        if (getter == null) {
+                            logger.warnElement(
+                                "Expected an annotation property to have a getter",
+                                it
+                            )
+                        } else {
+                            extractFunction(
+                                getter,
+                                id,
+                                listOf(),
+                                /* OLD: KE1
+                                extractBody = false,
+                                extractMethodAndParameterTypeAccesses =
+                                extractFunctionBodies,
+                                extractAnnotations = true,
+                                null
+                                 */
+                            )
+                                ?.also { functionLabel ->
+                                    tw.writeIsAnnotElem(functionLabel.cast())
+                                }
+                        }
+                    }
+                } else {
+                    try {
+                        c.declaredMemberScope.declarations.forEach {
+                            extractDeclaration(
+                                it,
+                                /*
+                                OLD: KE1
+                                extractPrivateMembers = extractPrivateMembers,
+                                extractFunctionBodies = extractFunctionBodies,
+                                extractAnnotations = true
+                                 */
+                            )
+                        }
+                        /*
+                        OLD: KE1
+                        if (extractStaticInitializer) extractStaticInitializer(c, { id })
+                        extractJvmStaticProxyMethods(
+                            c,
+                            id,
+                            extractPrivateMembers,
+                            extractFunctionBodies
+                        )
+                         */
+                    } catch (e: IllegalArgumentException) {
+                        // A Kotlin bug causes this to throw: https://youtrack.jetbrains.com/issue/KT-63847/K2-IllegalStateException-IrFieldPublicSymbolImpl-for-java.time-Clock.OffsetClock.offset0-is-already-bound
+                        // TODO: This should either be removed or log something, once the bug is fixed
+                    }
+                }
+            }
+            /*
+            OLD: KE1
+            if (c.isNonCompanionObject) {
+                // For `object MyObject { ... }`, the .class has an
+                // automatically-generated `public static final MyObject INSTANCE`
+                // field that may be referenced from Java code, and is used in our
+                // IrGetObjectValue support. We therefore need to fabricate it
+                // here.
+                val instance = useObjectClassInstance(c)
+                val type = useSimpleTypeClass(c, emptyList(), false)
+                tw.writeFields(instance.id, instance.name, type.javaResult.id, id, instance.id)
+                tw.writeFieldsKotlinType(instance.id, type.kotlinResult.id)
+                tw.writeHasLocation(instance.id, locId)
+                addModifiers(instance.id, "public", "static", "final")
+                tw.writeClass_object(id, instance.id)
+            }
+            */
+            if (c.classKind == KaClassKind.OBJECT) {
+                addModifiers(id, "static")
+            }
+            /*
+            OLD: KE1
+            if (extractFunctionBodies && needsObinitFunction(c)) {
+                extractObinitFunction(c, id)
+            }
+
+            extractClassModifiers(c, id)
+            extractClassSupertypes(
+                c,
+                id,
+                inReceiverContext = true
+            ) // inReceiverContext = true is specified to force extraction of member prototypes
+            // of base types
+
+            linesOfCode?.linesOfCodeInDeclaration(c, id)
+
+            val additionalAnnotations =
+                if (
+                    c.kind == ClassKind.ANNOTATION_CLASS &&
+                    c.origin != IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB
+                )
+                    metaAnnotationSupport.generateJavaMetaAnnotations(c, extractFunctionBodies)
+                else listOf()
+
+            extractAnnotations(
+                c,
+                c.annotations + additionalAnnotations,
+                id,
+                extractFunctionBodies
+            )
+
+            if (extractFunctionBodies && !c.isAnonymousObject && !c.isLocal)
+                externalClassExtractor.writeStubTrapFile(c)
+            */
+
+            return id
+        // }
+    }
+}
+
+data class ClassLabelResults(val classLabel: String, val shortName: String)
+
+// `args` can be null to describe a raw generic type.
+// For non-generic types it will be zero-length list.
+// TODO: raw types not yet implemented for KE2
+// Structure of argsIncludingOuterClasses: for Outer<OuterArg>.Inner<InnerArg1, InnerArg2>,
+// argsIncludingOuterClasses will be `[[OuterArg], [InnerArg1, InnerArg2]]`
+context(KaSession)
+private fun KotlinUsesExtractor.getClassLabel(
+    c: KaClassSymbol,
+    argsIncludingOuterClasses: List<List<KaTypeProjection>>?
+): ClassLabelResults {
+    val unquotedLabel = getUnquotedClassLabel(c, argsIncludingOuterClasses)
+    return ClassLabelResults("@\"class;${unquotedLabel.classLabel}\"", /* TODO , */ unquotedLabel.shortName)
+}
+
+context(KaSession)
+fun KotlinUsesExtractor.useClassSource(c: KaClassSymbol): Label<out DbClassorinterface> {
+    // For source classes, the label doesn't include any type arguments
+    val id = addClassLabel(c, listOf()).id
+    return id
+}
+
+/**
+ * Return a replacement class name and type arguments for cBeforeReplacement<argsIncludingOuterClassesBeforeReplacement>
+ *
+ * These could include replacing Android synthetic classes or Parcelize raw types that shouldn't appear in the database
+ * as they appear to the Kotlin compiler.
+ *
+ * TODO: verify how these sorts of classes appear via the analysis API
+ */
+private fun tryReplaceType(
+    cBeforeReplacement: KaClassSymbol,
+    argsIncludingOuterClassesBeforeReplacement: List<List<KaTypeProjection>>?
+): Pair<KaClassSymbol, List<List<KaTypeProjection>>?> {
+    return Pair(cBeforeReplacement, argsIncludingOuterClassesBeforeReplacement)
+    /*
+    OLD: KE1
+    val c = tryReplaceAndroidSyntheticClass(cBeforeReplacement)
+    val p = tryReplaceParcelizeRawType(c)
+    return Pair(p?.first ?: c, p?.second ?: argsIncludingOuterClassesBeforeReplacement)
+    */
+}
+
+private fun isExternalDeclaration(d: KaSymbol): Boolean {
+    return d.origin == KaSymbolOrigin.LIBRARY || d.origin == KaSymbolOrigin.JAVA_LIBRARY
+}
+
+private fun KotlinUsesExtractor.extractExternalClassLater(c: KaClassSymbol) {
+    /* OLD: KE1
+    dependencyCollector?.addDependency(c)
+     */
+    externalClassExtractor.extractLater(c)
+}
+
+
+private fun KotlinUsesExtractor.extractClassLaterIfExternal(c: KaClassSymbol) {
+    if (isExternalDeclaration(c)) {
+        extractExternalClassLater(c)
+    }
+}
+
+// `typeArgs` can be null to describe a raw generic type.
+// For non-generic types it will be zero-length list.
+// TODO: Should this be private?
+context(KaSession)
+fun KotlinUsesExtractor.addClassLabel(
+    cBeforeReplacement: KaClassSymbol, // TODO cBeforeReplacement: IrClass,
+    argsIncludingOuterClassesBeforeReplacement: List<List<KaTypeProjection>>?,
+    /*
+    OLD: KE1
+            inReceiverContext: Boolean = false
+    */
+): TypeResult<out DbClassorinterface> {
+    val replaced =
+        tryReplaceType(cBeforeReplacement, argsIncludingOuterClassesBeforeReplacement)
+    val replacedClass = replaced.first
+    val replacedArgsIncludingOuterClasses = replaced.second
+
+    val classLabelResult = getClassLabel(replacedClass, replacedArgsIncludingOuterClasses)
+
+    /*
+    OLD: KE1
+            var instanceSeenBefore = true
+    */
+
+    val classLabel: Label<out DbClassorinterface> =
+        tw.getLabelFor(classLabelResult.classLabel) {
+/*
+OLD: KE1
+                instanceSeenBefore = false
+
+*/
+            extractClassLaterIfExternal(replacedClass)
+        }
+
+    /*
+    OLD: KE1
+            if (
+                replacedArgsIncludingOuterClasses == null ||
+                    replacedArgsIncludingOuterClasses.isNotEmpty()
+            ) {
+                // If this is a generic type instantiation or a raw type then it has no
+                // source entity, so we need to extract it here
+                val shouldExtractClassDetails =
+                    inReceiverContext &&
+                        tw.lm.genericSpecialisationsExtracted.add(classLabelResult.classLabel)
+                if (!instanceSeenBefore || shouldExtractClassDetails) {
+                    this.withFileOfClass(replacedClass)
+                        .extractClassInstance(
+                            classLabel,
+                            replacedClass,
+                            replacedArgsIncludingOuterClasses,
+                            !instanceSeenBefore,
+                            shouldExtractClassDetails
+                        )
+                }
+            }
+
+            // TODO: This used to do the below, but that is a "type" thing rather than a "class" thing
+            val fqName = replacedClass.fqNameWhenAvailable
+            val signature =
+                if (replacedClass.isAnonymousObject) {
+                    null
+                } else if (fqName == null) {
+                    logger.error("Unable to find signature/fqName for ${replacedClass.name}")
+                    null
+                } else {
+                    fqName.asString()
+                }
+
+    */
+    return TypeResult(classLabel, /* TODO , signature, */ classLabelResult.shortName)
+}
+
+/*
+    OLD: KE1
+        /**
+         * This returns the `X` in c's label `@"class;X"`.
+         *
+         * `argsIncludingOuterClasses` can be null to describe a raw generic type. For non-generic types
+         * it will be zero-length list.
+         */
+    */
+context(KaSession)
+private fun KotlinUsesExtractor.getUnquotedClassLabel(
+    c: KaClassSymbol,
+    argsIncludingOuterClasses: List<List<KaTypeProjection>>?
+): ClassLabelResults {
+    val classId = c.classId
+    if (classId == null) {
+        TODO() // This is a local class
+    }
+    val pkg = classId.packageFqName.asString()
+    val cls = classId.shortClassName.asString()
+    val label =
+        /* OLD: KE1
+        if (c.isAnonymousObject) "{${useAnonymousClass(c).javaResult.id}}"
+        else
+         */
+            when (val parent = c.containingSymbol) {
+                is KaClassSymbol -> {
+                    "${getUnquotedClassLabel(parent, listOf()).classLabel}\$$cls"
+                }
+
+                is KaFunctionSymbol -> {
+                    "{${useFunction<DbMethod>(parent, listOf())}}.$cls"
+                }
+
+                is KaPropertySymbol -> {
+                    TODO()
+                    /* OLD: KE1
+                    "{${useField(parent)}}.$cls"
+                    */
+                }
+
+                else -> {
+                    if (pkg.isEmpty()) cls else "$pkg.$cls"
+                }
+            }
+
+            val typeArgLabels = argsIncludingOuterClasses?.flatten()?.map { getTypeArgumentLabel(it) }
+            val typeArgsShortName =
+                if (typeArgLabels == null) "<>"
+                else if (typeArgLabels.isEmpty()) ""
+                else
+                    typeArgLabels.takeLast((c as? KaNamedClassSymbol)?.typeParameters?.size ?: 0).joinToString(
+                        prefix = "<",
+                        postfix = ">",
+                        separator = ","
+                    ) {
+                        it.shortName
+                    }
+            val shortNamePrefix = if (c.name == null) "" else cls
+
+    return ClassLabelResults(
+        label + (typeArgLabels?.joinToString(separator = "") { ";{${it.id}}" } ?: "<>"),
+        shortNamePrefix + typeArgsShortName
+    )
+}

java/kotlin-extractor2/src/main/kotlin/entities/Function.kt

@@ -0,0 +1,816 @@
+package com.github.codeql
+
+import com.github.codeql.entities.extractTypeParameter
+import com.github.codeql.utils.getJvmName
+import com.github.codeql.utils.type
+import org.jetbrains.kotlin.analysis.api.KaSession
+import org.jetbrains.kotlin.analysis.api.symbols.*
+import org.jetbrains.kotlin.analysis.api.types.KaType
+import org.jetbrains.kotlin.analysis.api.types.KaTypeProjection
+import org.jetbrains.kotlin.load.java.JvmAbi
+import org.jetbrains.kotlin.psi.KtDeclarationWithBody
+
+/*
+         * There are some pairs of classes (e.g. `kotlin.Throwable` and
+         * `java.lang.Throwable`) which are really just 2 different names
+         * for the same class. However, we extract them as separate
+         * classes. When extracting `kotlin.Throwable`'s methods, if we
+         * looked up the parent ID ourselves, we would get as ID for
+         * `java.lang.Throwable`, which isn't what we want. So we have to
+         * allow it to be passed in.
+         *
+         * `maybeParameterList` can be supplied to override the function's
+         * value parameters; this is used for generating labels of overloads
+         * that omit one or more parameters that has a default value specified.
+
+@OptIn(ObsoleteDescriptorBasedAPI::class)
+*/
+context(KaSession)
+fun KotlinUsesExtractor.getFunctionLabel(
+    f: KaFunctionSymbol,
+    parentId: Label<out DbElement>,
+    classTypeArgsIncludingOuterClasses: List<KaTypeProjection>?,
+    maybeParameterList: List<KaValueParameterSymbol>? = null,
+): String =
+    getFunctionLabel(
+        /*
+        OLD: KE1
+                    f.parent,
+        */
+        parentId,
+        getFunctionShortName(f).nameInDB,
+        f is KaConstructorSymbol,
+        (maybeParameterList ?: f.valueParameters).map { it.type },
+        getAdjustedReturnType(f),
+        f.receiverParameter?.type,
+        getFunctionTypeParameters(f),
+        classTypeArgsIncludingOuterClasses,
+        /*
+        OLD: KE1
+                    overridesCollectionsMethodWithAlteredParameterTypes(f),
+                    getJavaCallable(f),
+                    !getInnermostWildcardSupppressionAnnotation(f)
+        */
+    )
+
+fun getAdjustedReturnType(f: KaFunctionSymbol): KaType {
+    return f.returnType
+    /* OLD: KE1
+    // Replace annotation val accessor types as needed:
+    (f as? IrSimpleFunction)?.correspondingPropertySymbol?.let {
+        if (isAnnotationClassProperty(it) && f == it.owner.getter) {
+            val replaced = kClassToJavaClass(f.returnType)
+            if (replaced != f.returnType) return replaced
+        }
+    }
+
+    // The return type of `java.util.concurrent.ConcurrentHashMap<K,V>.keySet/0` is defined as
+    // `Set<K>` in the stubs inside the Android SDK.
+    // This does not match the Java SDK return type: `ConcurrentHashMap.KeySetView<K,V>`, so
+    // it's adjusted here.
+    // This is a deliberate change in the Android SDK:
+    // https://github.com/AndroidSDKSources/android-sdk-sources-for-api-level-31/blob/2c56b25f619575bea12f9c5520ed2259620084ac/java/util/concurrent/ConcurrentHashMap.java#L1244-L1249
+    // The annotation on the source is not visible in the android.jar, so we can't make the
+    // change based on that.
+    // TODO: there are other instances of `dalvik.annotation.codegen.CovariantReturnType` in the
+    // Android SDK, we should handle those too if they cause DB inconsistencies
+    val parentClass = f.parentClassOrNull
+    if (
+        parentClass == null ||
+        parentClass.fqNameWhenAvailable?.asString() !=
+        "java.util.concurrent.ConcurrentHashMap" ||
+        getFunctionShortName(f).nameInDB != "keySet" ||
+        f.valueParameters.isNotEmpty() ||
+        f.returnType.classFqName?.asString() != "kotlin.collections.MutableSet"
+    ) {
+        return f.returnType
+    }
+
+    val otherKeySet =
+        parentClass.declarations.findSubType<IrFunction> {
+            it.name.asString() == "keySet" && it.valueParameters.size == 1
+        } ?: return f.returnType
+
+    return otherKeySet.returnType.codeQlWithHasQuestionMark(false)
+    */
+}
+
+/*
+OLD: KE1
+    /*
+     * This function actually generates the label for a function.
+     * Sometimes, a function is only generated by kotlinc when writing a
+     * class file, so there is no corresponding `IrFunction` for it.
+     * This function therefore takes all the constituent parts of a
+     * function instead.
+     */
+*/
+context(KaSession)
+fun KotlinUsesExtractor.getFunctionLabel(
+    /*
+    OLD: KE1
+            // The parent of the function; normally f.parent.
+            parent: IrDeclarationParent,
+    */
+    // The ID of the function's parent.
+    parentId: Label<out DbElement>,
+    // OLD: KE1: The name of the function; normally f.name.asString().
+    name: String,
+    // The types of the value parameters that the functions takes; normally
+    // f.valueParameters.map { it.type }.
+    isConstructor: Boolean,
+    parameterTypes: List<KaType>,
+    // The return type of the function; normally f.returnType.
+    returnType: KaType,
+    // The extension receiver of the function, if any; normally
+    // f.receiverParameter?.type.
+    extensionParamType: KaType?,
+    // The type parameters of the function. This does not include type parameters of enclosing
+    // classes.
+    functionTypeParameters: List<KaTypeParameterSymbol>,
+    // The type arguments of enclosing classes of the function.
+    classTypeArgsIncludingOuterClasses: List<KaTypeProjection>?,
+    /*
+    OLD: KE1
+            // If true, this method implements a Java Collections interface (Collection, Map or List)
+            // and may need
+            // parameter erasure to match the way this class will appear to an external consumer of the
+            // .class file.
+            overridesCollectionsMethod: Boolean,
+            // The Java signature of this callable, if known.
+            javaSignature: JavaMember?,
+            // If true, Java wildcards implied by Kotlin type parameter variance should be added by
+            // default to this function's value parameters' types.
+            // (Return-type wildcard addition is always off by default)
+            addParameterWildcardsByDefault: Boolean,
+    */
+    // The prefix used in the label. "callable", unless a property label is created, then it's
+    // "property".
+    prefix: String = "callable"
+): String {
+
+    val allParamTypes = listOfNotNull(extensionParamType) + parameterTypes
+
+    /* OLD: KE1
+    val substitutionMap =
+        classTypeArgsIncludingOuterClasses?.let { notNullArgs ->
+            if (notNullArgs.isEmpty()) {
+                null
+            } else {
+                val enclosingClass = getEnclosingClass(parent)
+                enclosingClass?.let { notNullClass ->
+                    makeTypeGenericSubstitutionMap(notNullClass, notNullArgs)
+                }
+            }
+        }
+     */
+    val getIdForFunctionLabel = { it: IndexedValue<KaType> ->
+        // Kotlin rewrites certain Java collections types adding additional generic constraints-- for example,
+        // Collection.remove(Object) becomes Collection.remove(Collection::E) in the Kotlin universe.
+        // If this has happened, erase the type again to get the correct Java signature.
+        val maybeAmendedForCollections = it.value
+        /* OLD: KE1
+            if (overridesCollectionsMethod)
+                eraseCollectionsMethodParameterType(it.value, name, it.index)
+            else it.value
+         */
+        // Add any wildcard types that the Kotlin compiler would add in the Java lowering of
+        // this function:
+        val withAddedWildcards = maybeAmendedForCollections
+            /* OLD: KE1
+            addJavaLoweringWildcards(
+                maybeAmendedForCollections,
+                addParameterWildcardsByDefault,
+                javaSignature?.let { sig -> getJavaValueParameterType(sig, it.index) }
+            )
+            */
+        // Now substitute any class type parameters in:
+        val maybeSubbed = withAddedWildcards
+        /* OLD: KE1
+            withAddedWildcards.substituteTypeAndArguments(
+                substitutionMap,
+                TypeContext.OTHER,
+                pluginContext
+            )
+         */
+        // Finally, mimic the Java extractor's behaviour by naming functions with type
+        // parameters for their erased types;
+        // those without type parameters are named for the generic type.
+        val maybeErased = if (functionTypeParameters.isEmpty()) maybeSubbed else erase(maybeSubbed)
+
+        "{${useType(maybeErased).javaResult.id}}"
+    }
+    val paramTypeIds =
+        allParamTypes
+            .withIndex()
+            .joinToString(separator = ",", transform = getIdForFunctionLabel)
+    val labelReturnType =
+        if (isConstructor)
+            builtinTypes.unit
+        else
+            erase(
+                /*
+                OLD: KE1
+                returnType.substituteTypeAndArguments(
+                    substitutionMap,
+                    TypeContext.RETURN,
+                    pluginContext
+                )
+                */
+                returnType
+            )
+    // Note that `addJavaLoweringWildcards` is not required here because the return type used to
+    // form the function
+    // label is always erased.
+    val returnTypeId = useType(labelReturnType, TypeContext.RETURN).javaResult.id
+    // This suffix is added to generic methods (and constructors) to match the Java extractor's
+    // behaviour.
+    // Comments in that extractor indicates it didn't want the label of the callable to clash
+    // with the raw
+    // method (and presumably that disambiguation is never needed when the method belongs to a
+    // parameterized
+    // instance of a generic class), but as of now I don't know when the raw method would be
+    // referred to.
+    val typeArgSuffix =
+        if (
+            functionTypeParameters.isNotEmpty() &&
+                classTypeArgsIncludingOuterClasses.isNullOrEmpty()
+        )
+            "<${functionTypeParameters.size}>"
+        else ""
+
+    return "@\"$prefix;{$parentId}.$name($paramTypeIds){$returnTypeId}$typeArgSuffix\""
+}
+
+data class FunctionNames(val nameInDB: String, val kotlinName: String)
+
+/*
+OLD: KE1
+private val IrDeclaration.isAnonymousFunction
+            get() = this is IrSimpleFunction && name == SpecialNames.NO_NAME_PROVIDED
+
+
+        @OptIn(ObsoleteDescriptorBasedAPI::class)
+        private fun getJvmModuleName(f: IrFunction) =
+            NameUtils.sanitizeAsJavaIdentifier(
+                getJvmModuleNameForDeserializedDescriptor(f.descriptor)
+                    ?: JvmCodegenUtil.getModuleName(pluginContext.moduleDescriptor)
+            )
+
+ */
+context(KaSession)
+fun KotlinUsesExtractor.getFunctionShortName(f: KaFunctionSymbol): FunctionNames {
+    if (f.isLocal)
+        return FunctionNames(
+            "invoke",
+            "invoke"
+        )
+    fun getSuffixIfInternal() = ""
+        /* OLD: KE1
+        if (
+            f.visibility == DescriptorVisibilities.INTERNAL &&
+                f !is IrConstructor &&
+                !(f.parent is IrFile || isExternalFileClassMember(f))
+        ) {
+            "\$" + getJvmModuleName(f)
+        } else {
+            ""
+        }
+         */
+
+    (f as? KaPropertyAccessorSymbol)?.let {
+        val propSymbol = it.containingSymbol!! as KaPropertySymbol // TODO: Drop or justify !!
+        val propName = propSymbol.name.asString()
+        val getter = propSymbol.getter
+        val setter = propSymbol.setter
+
+        /* OLD: KE1
+        if (it.owner.parentClassOrNull?.kind == ClassKind.ANNOTATION_CLASS) {
+            if (getter == null) {
+                logger.error(
+                    "Expected to find a getter for a property inside an annotation class"
+                )
+                return FunctionNames(propName, propName)
+            } else {
+                val jvmName = getJvmName(getter)
+                return FunctionNames(jvmName ?: propName, propName)
+            }
+        }
+         */
+
+        val maybeFunctionName =
+            when (f) {
+                getter -> JvmAbi.getterName(propName)
+                setter -> JvmAbi.setterName(propName)
+                else -> {
+                    logger.error(
+                        "Function has a corresponding property, but is neither the getter nor the setter"
+                    )
+                    null
+                }
+            }
+        maybeFunctionName?.let { defaultFunctionName ->
+            val suffix = ""
+            /* OLD: KE1
+                if (
+                    f.visibility == DescriptorVisibilities.PRIVATE &&
+                        f.origin == IrDeclarationOrigin.DEFAULT_PROPERTY_ACCESSOR
+                ) {
+                    "\$private"
+                } else {
+                    getSuffixIfInternal()
+                }
+             */
+            return FunctionNames(
+                getJvmName(f) ?: "$defaultFunctionName$suffix",
+                defaultFunctionName
+            )
+        }
+    }
+
+    // TODO: Justify or drop !!
+    val name = if (f is KaConstructorSymbol) f.containingSymbol!!.name!! else f.name!!
+    return FunctionNames(
+        getJvmName(f) ?: "${name.asString()}${getSuffixIfInternal()}",
+        name.identifier
+    )
+}
+
+/*
+OLD: KE1
+        /*
+         * This is the normal getFunctionLabel function to use. If you want
+         * to refer to the function in its source class then
+         * classTypeArgsIncludingOuterClasses should be null. Otherwise, it
+         * is the list of type arguments that need to be applied to its
+         * enclosing classes to get the instantiation that this function is
+         * in.
+         */
+        fun getFunctionLabel(
+            f: IrFunction,
+            classTypeArgsIncludingOuterClasses: List<IrTypeArgument>?
+        ): String? {
+            val parentId = useDeclarationParentOf(f, false, classTypeArgsIncludingOuterClasses, true)
+            if (parentId == null) {
+                logger.error("Couldn't get parent ID for function label")
+                return null
+            }
+            return getFunctionLabel(f, parentId, classTypeArgsIncludingOuterClasses)
+        }
+ */
+
+// This excludes class type parameters that show up in (at least) constructors' typeParameters list.
+context(KaSession)
+fun getFunctionTypeParameters(f: KaFunctionSymbol): List<KaTypeParameterSymbol> =
+    when (f) {
+        is KaConstructorSymbol -> f.typeParameters.filter { it.containingSymbol == f }
+        is KaNamedFunctionSymbol -> f.typeParameters
+        else -> listOf()
+    }
+
+context(KaSession)
+fun KotlinFileExtractor.extractFunction(
+    f: KaFunctionSymbol,
+    parentId: Label<out DbReftype>,
+    classTypeArgsIncludingOuterClasses: List<KaTypeProjection>?
+    /*
+    OLD: KE1
+            extractBody: Boolean,
+            extractMethodAndParameterTypeAccesses: Boolean,
+            extractAnnotations: Boolean,
+            typeSubstitution: TypeSubstitution?,
+    */
+): Label<out DbCallable> {
+    /*
+    OLD: KE1
+                val overriddenVisibility = null
+    */
+    return forceExtractFunction(
+        f,
+        parentId,
+        classTypeArgsIncludingOuterClasses,
+        /*
+        OLD: KE1
+                            extractBody,
+                            extractMethodAndParameterTypeAccesses,
+                            extractAnnotations,
+                            typeSubstitution,
+                            overriddenAttributes = overriddenVisibility
+        */
+    )
+    /*
+    OLD: KE1
+                    .also {
+                        // The defaults-forwarder function is a static utility, not a member, so we only
+                        // need to extract this for the unspecialised instance of this class.
+                        if (classTypeArgsIncludingOuterClasses.isNullOrEmpty())
+                            extractDefaultsFunction(
+                                f,
+                                parentId,
+                                extractBody,
+                                extractMethodAndParameterTypeAccesses
+                            )
+                        extractGeneratedOverloads(
+                            f,
+                            parentId,
+                            null,
+                            extractBody,
+                            extractMethodAndParameterTypeAccesses,
+                            typeSubstitution,
+                            classTypeArgsIncludingOuterClasses
+                        )
+                    }
+    */
+}
+
+// TODO: Can this be inlined?
+context(KaSession)
+private fun KotlinFileExtractor.forceExtractFunction(
+    f: KaFunctionSymbol,
+    parentId: Label<out DbReftype>,
+    classTypeArgsIncludingOuterClasses: List<KaTypeProjection>?,
+    /*
+    OLD: KE1
+            extractBody: Boolean,
+            extractMethodAndParameterTypeAccesses: Boolean,
+            extractAnnotations: Boolean,
+            typeSubstitution: TypeSubstitution?,
+            extractOrigin: Boolean = true,
+            overriddenAttributes: OverriddenFunctionAttributes? = null
+    */
+): Label<out DbCallable> {
+    with("function", f) {
+/*
+OLD: KE1
+            DeclarationStackAdjuster(f, overriddenAttributes).use {
+*/
+        /* OLD: KE1
+        val javaCallable = getJavaCallable(f)
+         */
+        getFunctionTypeParameters(f).mapIndexed { idx, tp ->
+            extractTypeParameter(
+                tp,
+                idx,
+                /* OLD: KE1
+                (javaCallable as? JavaTypeParameterListOwner)
+                    ?.typeParameters
+                    ?.getOrNull(idx)
+                 */
+            )
+        }
+
+
+        val id =
+            /*
+            OLD: KE1
+                                overriddenAttributes?.id
+                                    ?: // If this is a class that would ordinarily be replaced by a Java
+                                       // equivalent (e.g. kotlin.Map -> java.util.Map),
+                                    // don't replace here, really extract the Kotlin version:
+            */
+            useFunction<DbCallable>(
+                f,
+                parentId,
+                classTypeArgsIncludingOuterClasses,
+                /*
+                OLD: KE1
+                                            noReplace = true
+                */
+            )
+
+        /*
+        OLD: KE1
+                        val sourceDeclaration =
+                            overriddenAttributes?.sourceDeclarationId
+                                ?: if (typeSubstitution != null && overriddenAttributes?.id == null) {
+                                    val sourceFunId = useFunction<DbCallable>(f)
+                                    if (sourceFunId == null) {
+                                        logger.errorElement("Cannot get source ID for function", f)
+                                        id // TODO: This is wrong; we ought to just fail in this case
+                                    } else {
+                                        sourceFunId
+                                    }
+                                } else {
+                                    id
+                                }
+
+                        val extReceiver = f.extensionReceiverParameter
+                        // The following parameter order is correct, because member $default methods (where
+                        // the order would be [dispatchParam], [extensionParam], normalParams) are not
+                        // extracted here
+                        val fParameters =
+                            listOfNotNull(extReceiver) +
+                                (overriddenAttributes?.valueParameters ?: f.valueParameters)
+                        val paramTypes =
+                            fParameters.mapIndexed { i, vp ->
+                                extractValueParameter(
+                                    vp,
+                                    id,
+                                    i,
+                                    typeSubstitution,
+                                    sourceDeclaration,
+                                    classTypeArgsIncludingOuterClasses,
+                                    extractTypeAccess = extractMethodAndParameterTypeAccesses,
+                                    overriddenAttributes?.sourceLoc
+                                )
+                            }
+                        if (extReceiver != null) {
+                            val extendedType = paramTypes[0]
+                            tw.writeKtExtensionFunctions(
+                                id.cast<DbMethod>(),
+                                extendedType.javaResult.id,
+                                extendedType.kotlinResult.id
+                            )
+                        }
+        */
+
+        val paramsSignature = "()" // TODO:
+        /*
+        OLD: KE1
+                            paramTypes.joinToString(separator = ",", prefix = "(", postfix = ")") {
+                                signatureOrWarn(it.javaResult, f)
+                            }
+
+                        val adjustedReturnType =
+                            addJavaLoweringWildcards(
+                                getAdjustedReturnType(f),
+                                false,
+                                (javaCallable as? JavaMethod)?.returnType
+                            )
+                        val substReturnType =
+                            typeSubstitution?.let {
+                                it(adjustedReturnType, TypeContext.RETURN, pluginContext)
+                            } ?: adjustedReturnType
+        */
+        val functionSyntax = f.psi as? KtDeclarationWithBody
+        val locId = functionSyntax?.let {
+            tw.getLocation(functionSyntax)
+        } ?: tw.getWholeFileLocation()
+        /*
+        OLD: KE1
+                            overriddenAttributes?.sourceLoc
+                                ?: getLocation(f, classTypeArgsIncludingOuterClasses)
+
+                        if (f.symbol is IrConstructorSymbol) {
+                            val shortName =
+                                when {
+                                    adjustedReturnType.isAnonymous -> ""
+                                    typeSubstitution != null ->
+                                        useType(substReturnType).javaResult.shortName
+                                    else ->
+                                        adjustedReturnType.classFqName?.shortName()?.asString()
+                                            ?: f.name.asString()
+                                }
+                            extractConstructor(
+                                id.cast(),
+                                shortName,
+                                paramsSignature,
+                                parentId,
+                                sourceDeclaration.cast()
+                            )
+                        } else {
+                            val shortNames = getFunctionShortName(f)
+        */
+        val shortNames = getFunctionShortName(f)
+        val methodId = id.cast<DbMethod>()
+        extractMethod(
+            methodId,
+            /*
+            OLD: KE1
+                                    locId,
+            */
+            shortNames.nameInDB,
+            f.returnType, // OLD: KE1: substReturnType,
+            paramsSignature,
+            parentId,
+            /*
+            OLD: KE1
+                                    sourceDeclaration.cast(),
+                                    if (extractOrigin) f.origin else null,
+                                    extractMethodAndParameterTypeAccesses
+            */
+        )
+
+        /*
+        OLD: KE1
+                            if (shortNames.nameInDB != shortNames.kotlinName) {
+                                tw.writeKtFunctionOriginalNames(methodId, shortNames.kotlinName)
+                            }
+
+                            if (f.hasInterfaceParent() && f.body != null) {
+                                addModifiers(
+                                    methodId,
+                                    "default"
+                                ) // The actual output class file may or may not have this modifier,
+                                  // depending on the -Xjvm-default setting.
+                            }
+                        }
+        */
+
+        tw.writeHasLocation(id, locId)
+        val body = functionSyntax?.bodyExpression ?: functionSyntax?.bodyBlockExpression
+        if (body != null /* TODO && extractBody */) {
+            /*
+            OLD: KE1
+                                if (typeSubstitution != null)
+                                    logger.errorElement(
+                                        "Type substitution should only be used to extract a function prototype, not the body",
+                                        f
+                                    )
+            */
+            extractBody(body, id)
+        }
+
+        /*
+        OLD: KE1
+                        extractVisibility(f, id, overriddenAttributes?.visibility ?: f.visibility)
+
+                        if (f.isInline) {
+                            addModifiers(id, "inline")
+                        }
+                        if (f.shouldExtractAsStatic) {
+                            addModifiers(id, "static")
+                        }
+                        if (f is IrSimpleFunction && f.overriddenSymbols.isNotEmpty()) {
+                            addModifiers(id, "override")
+                        }
+                        if (f.isSuspend) {
+                            addModifiers(id, "suspend")
+                        }
+                        if (f.symbol !is IrConstructorSymbol) {
+                            when (overriddenAttributes?.modality ?: (f as? IrSimpleFunction)?.modality) {
+                                Modality.ABSTRACT -> addModifiers(id, "abstract")
+                                Modality.FINAL -> addModifiers(id, "final")
+                                else -> Unit
+                            }
+                        }
+
+                        linesOfCode?.linesOfCodeInDeclaration(f, id)
+
+                        if (extractAnnotations) {
+                            val extraAnnotations =
+                                if (f.symbol is IrConstructorSymbol) listOf()
+                                else
+                                    listOfNotNull(
+                                        getNullabilityAnnotation(
+                                            f.returnType,
+                                            f.origin,
+                                            f.annotations,
+                                            getJavaCallable(f)?.annotations
+                                        )
+                                    )
+                            extractAnnotations(
+                                f,
+                                f.annotations + extraAnnotations,
+                                id,
+                                extractMethodAndParameterTypeAccesses
+                            )
+                        }
+        */
+
+        return id
+        /*
+        OLD: KE1
+                    }
+        */
+    }
+}
+
+context(KaSession)
+fun KotlinFileExtractor.extractValueParameter(
+    id: Label<out DbParam>,
+    t: KaType,
+    name: String?,
+    locId: Label<DbLocation>,
+    parent: Label<out DbCallable>,
+    idx: Int,
+    paramSourceDeclaration: Label<out DbParam>,
+    isVararg: Boolean,
+    isNoinline: Boolean,
+    isCrossinline: Boolean
+): TypeResults {
+    val type = useType(t)
+    tw.writeParams(id, type.javaResult.id, idx, parent, paramSourceDeclaration)
+    tw.writeParamsKotlinType(id, type.kotlinResult.id)
+    tw.writeHasLocation(id, locId)
+    if (name != null) {
+        tw.writeParamName(id, name)
+    }
+    if (isVararg) {
+        tw.writeIsVarargsParam(id)
+    }
+    if (isNoinline) {
+        addModifiers(id, "noinline")
+    }
+    if (isCrossinline) {
+        addModifiers(id, "crossinline")
+    }
+    return type
+}
+
+// TODO: Can this be inlined?
+context(KaSession)
+private fun KotlinFileExtractor.extractMethod(
+    id: Label<out DbMethod>,
+    /*
+    OLD: KE1
+            locId: Label<out DbLocation>,
+    */
+    shortName: String,
+    returnType: KaType,
+    paramsSignature: String,
+    parentId: Label<out DbReftype>,
+    /*
+    OLD: KE1
+            sourceDeclaration: Label<out DbMethod>,
+            origin: IrDeclarationOrigin?,
+            extractTypeAccess: Boolean
+    */
+) {
+    val returnTypeResults = useType(returnType, TypeContext.RETURN)
+    tw.writeMethods(
+        id,
+        shortName,
+        "$shortName$paramsSignature",
+        returnTypeResults.javaResult.id,
+        parentId,
+        id, // OLD: KE1: sourceDeclaration
+    )
+    /*
+    OLD: KE1
+            tw.writeMethodsKotlinType(id, returnTypeResults.kotlinResult.id)
+            when (origin) {
+                IrDeclarationOrigin.GENERATED_DATA_CLASS_MEMBER ->
+                    tw.writeCompiler_generated(
+                        id,
+                        CompilerGeneratedKinds.GENERATED_DATA_CLASS_MEMBER.kind
+                    )
+                IrDeclarationOrigin.DEFAULT_PROPERTY_ACCESSOR ->
+                    tw.writeCompiler_generated(
+                        id,
+                        CompilerGeneratedKinds.DEFAULT_PROPERTY_ACCESSOR.kind
+                    )
+                IrDeclarationOrigin.ENUM_CLASS_SPECIAL_MEMBER ->
+                    tw.writeCompiler_generated(
+                        id,
+                        CompilerGeneratedKinds.ENUM_CLASS_SPECIAL_MEMBER.kind
+                    )
+            }
+            if (extractTypeAccess) {
+                extractTypeAccessRecursive(returnType, locId, id, -1)
+            }
+    */
+}
+
+context(KaSession)
+fun <T : DbCallable> KotlinUsesExtractor.useFunction(
+    f: KaFunctionSymbol,
+    classTypeArgsIncludingOuterClasses: List<KaTypeProjection>?,
+    /*
+    OLD: KE1
+            noReplace: Boolean = false
+    */
+): Label<out T> = useFunction(f, useDeclarationParentOf(f, true)!!, classTypeArgsIncludingOuterClasses)
+
+context(KaSession)
+fun <T : DbCallable> KotlinUsesExtractor.useFunction(
+    f: KaFunctionSymbol,
+    parentId: Label<out DbElement>,
+    classTypeArgsIncludingOuterClasses: List<KaTypeProjection>?,
+    /*
+    OLD: KE1
+            noReplace: Boolean = false
+    */
+): Label<out T> {
+    if (f.isLocal) {
+        val ids = tw.lm.getLocallyVisibleFunctionLabelMapping(f)
+        return ids.function.cast<T>()
+    }
+    val javaFun = f // TODO: kotlinFunctionToJavaEquivalent(f, noReplace)
+    return useFunction(f, javaFun, parentId, classTypeArgsIncludingOuterClasses)
+}
+
+context(KaSession)
+private fun <T : DbCallable> KotlinUsesExtractor.useFunction(
+    f: KaFunctionSymbol,
+    javaFun: KaFunctionSymbol,
+    parentId: Label<out DbElement>,
+    classTypeArgsIncludingOuterClasses: List<KaTypeProjection>?
+): Label<out T> {
+    val label = getFunctionLabel(javaFun, parentId, classTypeArgsIncludingOuterClasses)
+    val id: Label<T> =
+        tw.getLabelFor(label) {
+/*
+OLD: KE1
+                extractPrivateSpecialisedDeclaration(f, classTypeArgsIncludingOuterClasses)
+*/
+        }
+    /*
+    OLD: KE1
+            if (isExternalDeclaration(javaFun)) {
+                extractFunctionLaterIfExternalFileMember(javaFun)
+                extractExternalEnclosingClassLater(javaFun)
+            }
+    */
+    return id
+}

java/kotlin-extractor2/src/main/kotlin/entities/FunctionalInterface.kt

@@ -0,0 +1,563 @@
+import com.github.codeql.*
+import com.github.codeql.KotlinFileExtractor.ExprParent
+import com.github.codeql.utils.type
+import com.intellij.psi.PsiElement
+import org.jetbrains.kotlin.analysis.api.KaExperimentalApi
+import org.jetbrains.kotlin.analysis.api.KaSession
+import org.jetbrains.kotlin.analysis.api.symbols.*
+import org.jetbrains.kotlin.analysis.api.types.KaClassType
+import org.jetbrains.kotlin.analysis.api.types.KaFunctionType
+import org.jetbrains.kotlin.analysis.api.types.KaType
+import org.jetbrains.kotlin.builtins.StandardNames
+import org.jetbrains.kotlin.builtins.functions.BuiltInFunctionArity
+import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.psi.KtFunctionLiteral
+
+/**
+ * Extract a lambda expression as a generated anonymous class implementing
+ * the appropriate functional interface.
+ *
+ * Extract generated class:
+ * ```
+ * class C : Any, kotlin.FunctionI<T0,T1, ... TI, R> {
+ *   constructor() { super(); }
+ *   fun invoke(a0:T0, a1:T1, ... aI: TI): R { ... }
+ * }
+ * ```
+ * or in case of big arity lambdas
+ * ```
+ * class C : Any, kotlin.FunctionN<R> {
+ *   constructor() { super(); }
+ *   fun invoke(a0:T0, a1:T1, ... aI: TI): R { ... }
+ *   fun invoke(vararg args: Any?): R {
+ *     return invoke(args[0] as T0, args[1] as T1, ..., args[I] as TI)
+ *   }
+ * }
+ * ```
+ **/
+context(KaSession)
+fun KotlinFileExtractor.extractFunctionLiteral(
+    e: KtFunctionLiteral,
+    exprParent: ExprParent
+): Label<out DbExpr> {
+
+    val locId = tw.getLocation(e)
+    val functionSymbol = e.symbol
+    val ids = getLocallyVisibleFunctionLabels(functionSymbol)
+
+    val parameters = if (functionSymbol.isExtension) {
+        listOf(functionSymbol.receiverParameter!!) + functionSymbol.valueParameters
+    } else {
+        functionSymbol.valueParameters
+    }
+
+    val isBigArity = parameters.size >= BuiltInFunctionArity.BIG_ARITY
+    if (isBigArity) {
+        implementFunctionNInvoke(functionSymbol, ids, locId, parameters)
+    } else {
+        addModifiers(ids.function, "override")
+    }
+
+    val idLambdaExpr = tw.getFreshIdLabel<DbLambdaexpr>()
+    tw.writeExprs_lambdaexpr(
+        idLambdaExpr,
+        ids.type.javaResult.id,
+        exprParent.parent,
+        exprParent.idx
+    )
+    tw.writeExprsKotlinType(idLambdaExpr, ids.type.kotlinResult.id)
+    extractExprContext(idLambdaExpr, locId, exprParent.callable, exprParent.enclosingStmt)
+    tw.writeCallableBinding(idLambdaExpr, ids.constructor)
+
+    // todo: fix hard coded block body of lambda
+    tw.writeLambdaKind(idLambdaExpr, 1)
+
+    val functionType = getRealFunctionalInterfaceType(e.functionType as KaFunctionType)
+    if (!functionType.isFunctionType) {
+        logger.warnElement(
+            "Cannot find functional interface type for function expression",
+            e
+        )
+    } else {
+        val id =
+            extractGeneratedClass(
+                // We're adding this function as a member, and
+                // changing its name to `invoke` to implement
+                // `kotlin.FunctionX<,,,>.invoke(,,)`
+                functionSymbol,
+                e,
+                listOf(builtinTypes.any, functionType),
+                CompilerGeneratedKinds.CALLABLE_CLASS
+            )
+
+        /*
+        OLD: KE1
+        extractTypeAccessRecursive(
+            fnInterfaceType,
+            locId,
+            idLambdaExpr,
+            -3,
+            callable,
+            exprParent.enclosingStmt
+        )
+         */
+
+        tw.writeIsAnonymClass(id, idLambdaExpr)
+    }
+
+    return idLambdaExpr
+}
+
+context(KaSession)
+private fun KotlinFileExtractor.getRealFunctionalInterfaceType(typeFromApi: KaFunctionType): KaType {
+    if (typeFromApi.arity < BuiltInFunctionArity.BIG_ARITY) {
+        return typeFromApi
+    }
+
+    // TODO: the below doesn't work, see https://youtrack.jetbrains.com/issue/KT-73421/
+    return buildClassType(
+        ClassId(
+            FqName("kotlin.jvm.functions"),
+            Name.identifier("FunctionN")
+        )
+    ) {
+        argument(typeFromApi.typeArguments.last().type!!)
+    }
+}
+
+/**
+ * This function generates an implementation for `fun kotlin.FunctionN<R>.invoke(vararg args: Any?): R`
+ *
+ * The following body is added:
+ * ```
+ * fun invoke(vararg a0: Any?): R {
+ *   return invoke(a0[0] as T0, a0[1] as T1, ..., a0[I] as TI)
+ * }
+ * ```
+ * */
+context(KaSession)
+private fun KotlinFileExtractor.implementFunctionNInvoke(
+    lambda: KaFunctionSymbol,
+    ids: LocallyVisibleFunctionLabels,
+    locId: Label<DbLocation>,
+    parameters: List<KaParameterSymbol>
+) {
+    val funLabels =
+        addFunctionNInvoke(
+            tw.getFreshIdLabel(),
+            lambda.returnType,
+            ids.type.javaResult.id.cast<DbReftype>(),
+            locId
+        )
+
+    // Return
+    val retId = tw.getFreshIdLabel<DbReturnstmt>()
+    tw.writeStmts_returnstmt(retId, funLabels.blockId, 0, funLabels.methodId)
+    tw.writeHasLocation(retId, locId)
+
+    // Call to original `invoke`:
+    val callId = tw.getFreshIdLabel<DbMethodaccess>()
+    val callType = useType(lambda.returnType)
+    tw.writeExprs_methodaccess(callId, callType.javaResult.id, retId, 0)
+    tw.writeExprsKotlinType(callId, callType.kotlinResult.id)
+    extractExprContext(callId, locId, funLabels.methodId, retId)
+    tw.writeCallableBinding(callId, ids.function)
+
+    // this access
+    // OLD: KE1
+    // extractThisAccess(ids.type, funLabels.methodId, callId, -1, retId, locId)
+
+    addArgumentsToInvocationInInvokeNBody(
+        parameters.map { it.type },
+        funLabels,
+        retId,
+        callId,
+        locId
+    )
+}
+
+private data class FunctionLabels(
+    val methodId: Label<DbMethod>,
+    val blockId: Label<DbBlock>,
+    val parameters: List<Pair<Label<DbParam>, TypeResults>>
+)
+
+/**
+ * Adds a function `invoke(a: Any[])` with the specified return type to the class identified by
+ * `parentId`.
+ */
+context(KaSession)
+private fun KotlinFileExtractor.addFunctionNInvoke(
+    methodId: Label<DbMethod>,
+    returnType: KaType,
+    parentId: Label<out DbReftype>,
+    locId: Label<DbLocation>
+): FunctionLabels {
+    return addFunctionInvoke(
+        methodId,
+        listOf(nullableAnyArrayType),
+        returnType,
+        parentId,
+        locId
+    )
+}
+
+context(KaSession)
+private val nullableAnyArrayType: KaType
+    get() = buildClassType(ClassId.topLevel(StandardNames.FqNames.array.toSafe())) {
+        argument(builtinTypes.nullableAny)
+    }
+
+/**
+ * Adds a function named `invoke` with the specified parameter types and return type to the
+ * class identified by `parentId`.
+ */
+context(KaSession)
+private fun KotlinFileExtractor.addFunctionInvoke(
+    methodId: Label<DbMethod>,
+    parameterTypes: List<KaType>,
+    returnType: KaType,
+    parentId: Label<out DbReftype>,
+    locId: Label<DbLocation>
+): FunctionLabels {
+    return addFunctionManual(
+        methodId,
+        "invoke",
+        parameterTypes,
+        returnType,
+        parentId,
+        locId
+    )
+}
+
+/**
+ * Extracts a function with the given name, parameter types, return type, containing type, and
+ * location.
+ */
+context(KaSession)
+private fun KotlinFileExtractor.addFunctionManual(
+    methodId: Label<DbMethod>,
+    name: String,
+    parameterTypes: List<KaType>,
+    returnType: KaType,
+    parentId: Label<out DbReftype>,
+    locId: Label<DbLocation>
+): FunctionLabels {
+
+    val parameters =
+        parameterTypes.mapIndexed { idx, p ->
+            val paramId = tw.getFreshIdLabel<DbParam>()
+            val paramType =
+                extractValueParameter(
+                    paramId,
+                    p,
+                    "a$idx",
+                    locId,
+                    methodId,
+                    idx,
+                    paramId,
+                    isVararg = false,
+                    isNoinline = false,
+                    isCrossinline = false
+                )
+
+            Pair(paramId, paramType)
+        }
+
+    /* OLD: KE1
+    val paramsSignature =
+        parameters.joinToString(separator = ",", prefix = "(", postfix = ")") {
+            signatureOrWarn(it.second.javaResult, declarationStack.tryPeek()?.first)
+        }
+     */
+    val paramsSignature = "()" // TODO
+
+    val rt = useType(returnType, TypeContext.RETURN)
+    tw.writeMethods(
+        methodId,
+        name,
+        "$name$paramsSignature",
+        rt.javaResult.id,
+        parentId,
+        methodId
+    )
+    tw.writeMethodsKotlinType(methodId, rt.kotlinResult.id)
+    tw.writeHasLocation(methodId, locId)
+
+    addModifiers(methodId, "public")
+    addModifiers(methodId, "override")
+
+    return FunctionLabels(methodId, extractBlockBody(methodId, locId), parameters)
+}
+
+/**
+ * Adds the arguments to the method call inside `invoke(a0: Any[])`. Each argument is an array
+ * access with a cast:
+ * ```
+ * fun invoke(a0: Any[]) : T {
+ *   return fn(a0[0] as T0, a0[1] as T1, ...)
+ * }
+ * ```
+ */
+context(KaSession)
+private fun KotlinFileExtractor.addArgumentsToInvocationInInvokeNBody(
+    parameterTypes: List<KaType>,   // list of parameter types
+    funLabels: FunctionLabels,      // already generated labels for the function definition
+    enclosingStmtId: Label<out DbStmt>,     // label for the enclosing statement (return)
+    exprParentId: Label<out DbExprparent>,  // label for the expression parent (call)
+    locId: Label<DbLocation>,       // label for the location of all generated items
+    firstArgumentOffset: Int =
+        0,              // 0 or 1, the index used for the first argument. 1 in case an extension parameter is already accessed at index 0
+    useFirstArgAsDispatch: Boolean =
+        false,          // true if the first argument should be used as the dispatch receiver
+    dispatchReceiverIdx: Int =
+        -1              // index of the dispatch receiver. -1 in case of functions, -2 in case of constructors
+) {
+    val argsParamType = nullableAnyArrayType
+    val argsType = useType(argsParamType)
+    val anyNType = useType(builtinTypes.nullableAny)
+
+    val dispatchIdxOffset = if (useFirstArgAsDispatch) 1 else 0
+
+    for ((pIdx, pType) in parameterTypes.withIndex()) {
+        // `a0[i] as Ti` is generated below for each parameter
+
+        val childIdx =
+            if (pIdx == 0 && useFirstArgAsDispatch) {
+                dispatchReceiverIdx
+            } else {
+                pIdx + firstArgumentOffset - dispatchIdxOffset
+            }
+
+        // cast: `(Ti)a0[i]`
+        val castId = tw.getFreshIdLabel<DbCastexpr>()
+        val type = useType(pType)
+        tw.writeExprs_castexpr(castId, type.javaResult.id, exprParentId, childIdx)
+        tw.writeExprsKotlinType(castId, type.kotlinResult.id)
+        extractExprContext(castId, locId, funLabels.methodId, enclosingStmtId)
+
+        // type access `Ti`
+        // TODO: extractTypeAccessRecursive(pType, locId, castId, 0, funLabels.methodId, enclosingStmtId)
+
+        // element access: `a0[i]`
+        val arrayAccessId = tw.getFreshIdLabel<DbArrayaccess>()
+        tw.writeExprs_arrayaccess(arrayAccessId, anyNType.javaResult.id, castId, 1)
+        tw.writeExprsKotlinType(arrayAccessId, anyNType.kotlinResult.id)
+        extractExprContext(arrayAccessId, locId, funLabels.methodId, enclosingStmtId)
+
+        // parameter access: `a0`
+        val argsAccessId = tw.getFreshIdLabel<DbVaraccess>()
+        tw.writeExprs_varaccess(argsAccessId, argsType.javaResult.id, arrayAccessId, 0)
+        tw.writeExprsKotlinType(argsAccessId, argsType.kotlinResult.id)
+        extractExprContext(argsAccessId, locId, funLabels.methodId, enclosingStmtId)
+        tw.writeVariableBinding(argsAccessId, funLabels.parameters.first().first)
+
+        // index access: `i`
+        extractConstantInteger(
+            pIdx.toString(),
+            builtinTypes.int,
+            pIdx,
+            locId,
+            arrayAccessId,
+            1,
+            funLabels.methodId,
+            enclosingStmtId
+        )
+    }
+}
+
+
+/**
+ * Gets the labels for functions belonging to
+ * - local functions, and
+ * - lambdas.
+ */
+private fun KotlinFileExtractor.getLocallyVisibleFunctionLabels(f: KaAnonymousFunctionSymbol): LocallyVisibleFunctionLabels {
+    if (!f.isLocal) {
+        logger.error("Extracting a non-local function as a local one")
+    }
+
+    return tw.lm.getOrAddLocallyVisibleFunctionLabelMapping(f) {
+        val classId = tw.getFreshIdLabel<DbClassorinterface>()
+        val javaResult = TypeResult(classId, /* "TODO", */ "")
+        val kotlinTypeId =
+            tw.getLabelFor<DbKt_class_type>("@\"kt_class;{$classId}\"") {
+                tw.writeKt_class_types(it, classId)
+            }
+        val kotlinResult = TypeResult(kotlinTypeId, /* "TODO", */ "")
+
+        LocallyVisibleFunctionLabels(
+            type = TypeResults(javaResult, kotlinResult),
+            constructor = tw.getFreshIdLabel(),
+            constructorBlock = tw.getFreshIdLabel(),
+            function = tw.getFreshIdLabel()
+        )
+    }
+}
+
+/**
+ * Extracts the class around a local function or a lambda. The superclass must have a no-arg
+ * constructor.
+ */
+context(KaSession)
+private fun KotlinFileExtractor.extractGeneratedClass(
+    localFunction: KaFunctionSymbol,
+    elementToReportOn: PsiElement,
+    superTypes: List<KaType>,
+    compilerGeneratedKindOverride: CompilerGeneratedKinds
+): Label<out DbClassorinterface> {
+    val ids = tw.lm.getLocallyVisibleFunctionLabelMapping(localFunction)
+
+    val id =
+        extractGeneratedClass(
+            ids,
+            superTypes,
+            tw.getLocation(elementToReportOn),
+            elementToReportOn,
+            compilerGeneratedKindOverride = compilerGeneratedKindOverride
+            /*
+            OLD: KE1
+            localFunction.parent,
+             */
+        )
+
+    // Extract local function as a member
+    extractFunction(
+        localFunction,
+        id,
+        listOf()
+        /*
+        OLD: KE1
+        extractBody = true,
+        extractMethodAndParameterTypeAccesses = true,
+        extractAnnotations = false,
+        null,
+         */
+    )
+
+    return id
+}
+
+/** Extracts the class around a local function, a lambda, or a function reference. */
+context(KaSession)
+@OptIn(KaExperimentalApi::class)
+private fun KotlinFileExtractor.extractGeneratedClass(
+    ids: GeneratedClassLabels,
+    superTypes: List<KaType>,
+    locId: Label<DbLocation>,
+    elementToReportOn: PsiElement,
+    compilerGeneratedKindOverride: CompilerGeneratedKinds
+    /*
+    OLD: KE1
+    declarationParent: IrDeclarationParent,
+     */
+): Label<out DbClassorinterface> {
+    // Write class
+    val id = ids.type.javaResult.id.cast<DbClassorinterface>()
+    val pkgId = extractPackage("")
+    tw.writeClasses_or_interfaces(id, "", pkgId, id)
+    tw.writeCompiler_generated(id, compilerGeneratedKindOverride.kind)
+    tw.writeHasLocation(id, locId)
+
+    // Extract constructor
+    val unitType = useType(builtinTypes.unit/*TODO , TypeContext.RETURN*/)
+    tw.writeConstrs(ids.constructor, "", "", unitType.javaResult.id, id, ids.constructor)
+    tw.writeConstrsKotlinType(ids.constructor, unitType.kotlinResult.id)
+    tw.writeHasLocation(ids.constructor, locId)
+    addModifiers(ids.constructor, "public")
+
+    // Constructor body
+    val constructorBlockId = ids.constructorBlock
+    tw.writeStmts_block(constructorBlockId, ids.constructor, 0, ids.constructor)
+    tw.writeHasLocation(constructorBlockId, locId)
+
+    // Super call
+    val baseClass = superTypes.first() as? KaClassType
+    if ((baseClass?.symbol as? KaClassSymbol)?.classKind != KaClassKind.CLASS) {
+        logger.warnElement("Cannot find base class", elementToReportOn)
+    } else {
+        val baseConstructor =
+            baseClass.scope?.declarationScope?.constructors?.find {
+                it.valueParameters.isEmpty()
+            }
+        if (baseConstructor == null) {
+            logger.warnElement("Cannot find base constructor", elementToReportOn)
+        } else {
+            val baseConstructorParentId = useDeclarationParentOf(baseConstructor, false)
+            if (baseConstructorParentId == null) {
+                logger.errorElement("Cannot find base constructor ID", elementToReportOn)
+            } else {
+                val baseConstructorId = useFunction<DbConstructor>(baseConstructor, baseConstructorParentId, listOf())
+                val superCallId = tw.getFreshIdLabel<DbSuperconstructorinvocationstmt>()
+                tw.writeStmts_superconstructorinvocationstmt(
+                    superCallId,
+                    constructorBlockId,
+                    0,
+                    ids.constructor
+                )
+
+                tw.writeHasLocation(superCallId, locId)
+                tw.writeCallableBinding(superCallId.cast<DbCaller>(), baseConstructorId)
+            }
+        }
+    }
+
+    addModifiers(id, "final")
+    addModifiers(id, "private")
+    //OLD: KE1
+    //    extractClassSupertypes(
+    //        superTypes,
+    //        //listOf(),
+    //        id,
+    //        //isInterface = false,
+    //        //inReceiverContext = true
+    //    )
+
+    // TODO: OLD KE1:
+    // extractEnclosingClass(declarationParent, id, null, locId, listOf())
+
+    return id
+}
+
+/**
+ * Class to hold labels for generated classes around local functions, lambdas, function
+ * references, and property references.
+ */
+open class GeneratedClassLabels(
+    val type: TypeResults,
+    val constructor: Label<DbConstructor>,
+    val constructorBlock: Label<DbBlock>
+)
+
+/**
+ * Class to hold labels generated for locally visible functions, such as
+ * - local functions,
+ * - lambdas, and
+ * - wrappers around function references.
+ */
+class LocallyVisibleFunctionLabels(
+    type: TypeResults,
+    constructor: Label<DbConstructor>,
+    constructorBlock: Label<DbBlock>,
+    val function: Label<DbMethod>
+) : GeneratedClassLabels(type, constructor, constructorBlock)
+
+
+private enum class CompilerGeneratedKinds(val kind: Int) {
+    // OLD: KE1
+    //    DECLARING_CLASSES_OF_ADAPTER_FUNCTIONS(1),
+    //    GENERATED_DATA_CLASS_MEMBER(2),
+    //    DEFAULT_PROPERTY_ACCESSOR(3),
+    //    CLASS_INITIALISATION_METHOD(4),
+    //    ENUM_CLASS_SPECIAL_MEMBER(5),
+    //    DELEGATED_PROPERTY_GETTER(6),
+    //    DELEGATED_PROPERTY_SETTER(7),
+    //    JVMSTATIC_PROXY_METHOD(8),
+    //    JVMOVERLOADS_METHOD(9),
+    //    DEFAULT_ARGUMENTS_METHOD(10),
+    //    INTERFACE_FORWARDER(11),
+    //    ENUM_CONSTRUCTOR_ARGUMENT(12),
+    CALLABLE_CLASS(13),
+}

java/kotlin-extractor2/src/main/kotlin/entities/MethodCall.kt

@@ -0,0 +1,863 @@
+package com.github.codeql
+
+import com.github.codeql.KotlinFileExtractor.ExprParent
+import org.jetbrains.kotlin.analysis.api.KaSession
+import org.jetbrains.kotlin.analysis.api.resolution.KaSimpleFunctionCall
+import org.jetbrains.kotlin.analysis.api.resolution.symbol
+import org.jetbrains.kotlin.analysis.api.symbols.KaFunctionSymbol
+import org.jetbrains.kotlin.analysis.api.types.KaClassType
+import org.jetbrains.kotlin.analysis.api.types.KaType
+import org.jetbrains.kotlin.analysis.api.types.KaTypeProjection
+import org.jetbrains.kotlin.psi.KtCallExpression
+import org.jetbrains.kotlin.psi.KtExpression
+import org.jetbrains.kotlin.psi.KtParenthesizedExpression
+import org.jetbrains.kotlin.psi.KtQualifiedExpression
+import org.jetbrains.kotlin.psi.KtSafeQualifiedExpression
+import org.jetbrains.kotlin.utils.mapToIndex
+
+context(KaSession)
+fun KotlinFileExtractor.extractMethodCall(
+    call: KtCallExpression,
+    parent: ExprParent
+): Label<out DbExpr> {
+    val callTarget = call.resolveCallTarget() as? KaSimpleFunctionCall?
+    val target = callTarget?.symbol
+    val argMapping = callTarget?.argumentMapping
+
+    if (target == null || argMapping == null) TODO()
+
+    val parameterIndexMap = target.valueParameters.mapToIndex()
+
+    // TODO: we need to handle
+    // - arguments passed to vararg parameters, in which case there can be multiple (idx, expr) pairs with the same idx.
+    // - missing arguments due to default parameter values, in which case some indices are missing.
+    val args = call.valueArguments
+        .map { arg ->
+            val expr = arg.getArgumentExpression()
+
+            // `argMapping` seems to drill into parenthesized expressions
+            // TODO: improve this based on https://youtrack.jetbrains.com/issue/KT-73184
+            val (childExpr, _) = drillIntoParenthesizedExpression(expr!!)
+
+            val p = argMapping[childExpr]
+            if (p == null) {
+                TODO("This is unexpected, no parameter was found for the argument")
+            }
+            val idx = parameterIndexMap[p.symbol]
+            if (idx == null) {
+                TODO("This is unexpected, we couldn't find the parameter that the argument was mapped to")
+            }
+            Pair(idx, expr)
+        }
+        .sortedBy { p -> p.first }
+        .map { p -> p.second }
+
+    val callQualifiedParent = call.parent as? KtQualifiedExpression
+    val qualifier =
+        if (callQualifiedParent?.selectorExpression == call) callQualifiedParent.receiverExpression else null
+    val extensionReceiver = if (target.isExtension) qualifier else null
+    val dispatchReceiver = if (!target.isExtension) qualifier else null
+
+    val callId = extractRawMethodAccess(
+        target,
+        tw.getLocation(call),
+        call.expressionType!!,
+        parent.callable,
+        parent.parent,
+        parent.idx,
+        parent.enclosingStmt,
+        dispatchReceiver,
+        extensionReceiver,
+        args
+    )
+
+    if (call.parent is KtSafeQualifiedExpression) {
+        tw.writeKtSafeAccess(callId)
+    }
+
+    return callId
+}
+
+context(KaSession)
+private fun KotlinFileExtractor.extractCallValueArguments(
+    callId: Label<out DbExprparent>,
+    valueArguments: List<KtExpression?>,
+    enclosingStmt: Label<out DbStmt>,
+    enclosingCallable: Label<out DbCallable>,
+    idxOffset: Int,
+    // extractVarargAsArray: Boolean = false // OLD KE1
+) {
+    var i = 0
+    valueArguments.forEach { arg ->
+        if (arg != null) {
+            /* OLD KE1:
+            if (arg is IrVararg && !extractVarargAsArray) {
+                arg.elements.forEachIndexed { varargNo, vararg ->
+                    extractVarargElement(
+                        vararg,
+                        enclosingCallable,
+                        callId,
+                        i + idxOffset + varargNo,
+                        enclosingStmt
+                    )
+                }
+                i += arg.elements.size
+            } else {
+            */
+            extractExpressionExpr(
+                arg,
+                enclosingCallable,
+                callId,
+                (i++) + idxOffset,
+                enclosingStmt
+            )
+            //}
+        }
+    }
+}
+
+context(KaSession)
+private fun KotlinFileExtractor.getCalleeMethodId(callTarget: KaFunctionSymbol, classTypeArgsIncludingOuterClasses: List<KaTypeProjection>?): Label<out DbCallable> {
+    // TODO: is the below `useDeclarationParentOf` call correct?
+    // TODO: what should happen if the parent label is null?
+    return useFunction<DbCallable>(callTarget, useDeclarationParentOf(callTarget, false)!!, classTypeArgsIncludingOuterClasses)
+}
+
+context(KaSession)
+fun KotlinFileExtractor.extractRawMethodAccess(
+    callTarget: KaFunctionSymbol,
+    locId: Label<DbLocation>,
+    returnType: KaType,
+    enclosingCallable: Label<out DbCallable>,
+    callsiteParent: Label<out DbExprparent>,
+    childIdx: Int,
+    enclosingStmt: Label<out DbStmt>,
+    dispatchReceiver: KtExpression?,
+    extensionReceiver: KtExpression?,
+    valueArguments: List<KtExpression?>,
+    /* OLD KE1
+    syntacticCallTarget: IrFunction,
+    locId: Label<DbLocation>,
+    returnType: IrType,
+    enclosingCallable: Label<out DbCallable>,
+    callsiteParent: Label<out DbExprparent>,
+    childIdx: Int,
+    enclosingStmt: Label<out DbStmt>,
+    nValueArguments: Int,
+    extractValueArguments: (Label<out DbExpr>, Int) -> Unit,
+    drType: IrType?,
+    extractDispatchReceiver: ((Label<out DbExpr>) -> Unit)?,
+    extractExtensionReceiver: ((Label<out DbExpr>) -> Unit)?,
+    typeArguments: List<IrType> = listOf(),
+    extractClassTypeArguments: Boolean = false,
+    superQualifierSymbol: IrClassSymbol? = null
+     */
+): Label<DbMethodaccess> {
+    /* OLD KE1:
+    val callTarget = getCalleeRealOverrideTarget(syntacticCallTarget)
+    val methodId = getCalleeMethodId(callTarget, drType, extractClassTypeArguments)
+    if (methodId == null) {
+        logger.warn("No method to bind call to for raw method access")
+    }
+     */
+
+    val methodId = getCalleeMethodId(callTarget, (dispatchReceiver?.expressionType as? KaClassType)?.typeArguments)
+
+    val id =
+        extractMethodAccessWithoutArgs(
+            returnType,
+            locId,
+            enclosingCallable,
+            callsiteParent,
+            childIdx,
+            enclosingStmt,
+            methodId
+        )
+
+    /* OLD KE1:
+    // type arguments at index -2, -3, ...
+    extractTypeArguments(typeArguments, locId, id, enclosingCallable, enclosingStmt, -2, true)
+
+    if (callTarget.isLocalFunction()) {
+        extractNewExprForLocalFunction(
+            getLocallyVisibleFunctionLabels(callTarget),
+            id,
+            locId,
+            enclosingCallable,
+            enclosingStmt
+        )
+    } else if (callTarget.shouldExtractAsStatic) {
+        extractStaticTypeAccessQualifier(
+            callTarget,
+            id,
+            locId,
+            enclosingCallable,
+            enclosingStmt
+        )
+    } else if (superQualifierSymbol != null) {
+        extractSuperAccess(
+            superQualifierSymbol.typeWith(),
+            enclosingCallable,
+            id,
+            -1,
+            enclosingStmt,
+            locId
+        )
+    } else if (extractDispatchReceiver != null) {
+        extractDispatchReceiver(id)
+    }
+
+    val idxOffset = if (extractExtensionReceiver != null) 1 else 0
+
+    val isBigArityFunctionInvoke =
+        drType is IrSimpleType &&
+                isFunctionInvoke(callTarget, drType) &&
+                drType.arguments.size > BuiltInFunctionArity.BIG_ARITY
+
+    val argParent =
+        if (isBigArityFunctionInvoke) {
+            extractArrayCreationWithInitializer(
+                id,
+                nValueArguments + idxOffset,
+                locId,
+                enclosingCallable,
+                enclosingStmt
+            )
+        } else {
+            id
+        }
+
+    if (extractExtensionReceiver != null) {
+        extractExtensionReceiver(argParent)
+    }
+
+    extractValueArguments(argParent, idxOffset)
+
+    */
+
+    if (dispatchReceiver != null) {
+        extractExpressionExpr(dispatchReceiver, enclosingCallable, id, -1, enclosingStmt)
+    }
+    if (extensionReceiver != null) {
+        extractExpressionExpr(extensionReceiver, enclosingCallable, id, 0, enclosingStmt)
+    }
+    val idxOffset = if (extensionReceiver != null) 1 else 0
+    extractCallValueArguments(id, valueArguments, enclosingStmt, enclosingCallable, idxOffset)
+
+    return id
+}
+
+context(KaSession)
+private fun KotlinFileExtractor.extractMethodAccessWithoutArgs(
+    returnType: KaType,
+    locId: Label<DbLocation>,
+    enclosingCallable: Label<out DbCallable>,
+    callsiteParent: Label<out DbExprparent>,
+    childIdx: Int,
+    enclosingStmt: Label<out DbStmt>,
+    methodLabel: Label<out DbCallable>?
+) =
+    tw.getFreshIdLabel<DbMethodaccess>().also { id ->
+        val type = useType(returnType)
+
+        tw.writeExprs_methodaccess(id, type.javaResult.id, callsiteParent, childIdx)
+        tw.writeExprsKotlinType(id, type.kotlinResult.id)
+        extractExprContext(id, locId, enclosingCallable, enclosingStmt)
+
+        // The caller should have warned about this before, so we don't repeat the warning here.
+        if (methodLabel != null) tw.writeCallableBinding(id, methodLabel)
+    }
+
+/*
+OLD: KE1
+
+private fun getDeclaringTypeArguments(
+            callTarget: IrFunction,
+            receiverType: IrSimpleType
+        ): List<IrTypeArgument> {
+            val declaringType = callTarget.parentAsClass
+            val receiverClass = receiverType.classifier.owner as? IrClass ?: return listOf()
+            val ancestorTypes = ArrayList<IrSimpleType>()
+
+            // KFunctionX doesn't implement FunctionX on versions before 1.7.0:
+            if (
+                (callTarget.name.asString() == "invoke") &&
+                    (receiverClass.fqNameWhenAvailable
+                        ?.asString()
+                        ?.startsWith("kotlin.reflect.KFunction") == true) &&
+                    (callTarget.parentClassOrNull
+                        ?.fqNameWhenAvailable
+                        ?.asString()
+                        ?.startsWith("kotlin.Function") == true)
+            ) {
+                return receiverType.arguments
+            }
+
+            // Populate ancestorTypes with the path from receiverType's class to its ancestor,
+            // callTarget's declaring type.
+            fun walkFrom(c: IrClass): Boolean {
+                if (declaringType == c) return true
+                else {
+                    c.superTypes.forEach {
+                        val ancestorClass =
+                            (it as? IrSimpleType)?.classifier?.owner as? IrClass ?: return false
+                        ancestorTypes.add(it)
+                        if (walkFrom(ancestorClass)) return true else ancestorTypes.pop()
+                    }
+                    return false
+                }
+            }
+
+            // If a path was found, repeatedly substitute types to get the corresponding specialisation
+            // of that ancestor.
+            if (!walkFrom(receiverClass)) {
+                logger.errorElement(
+                    "Failed to find a class declaring ${callTarget.name} starting at ${receiverClass.name}",
+                    callTarget
+                )
+                return listOf()
+            } else {
+                var subbedType: IrSimpleType = receiverType
+                ancestorTypes.forEach {
+                    val thisClass = subbedType.classifier.owner
+                    if (thisClass !is IrClass) {
+                        logger.errorElement(
+                            "Found ancestor with unexpected type ${thisClass.javaClass}",
+                            callTarget
+                        )
+                        return listOf()
+                    }
+                    val itSubbed =
+                        it.substituteTypeArguments(thisClass.typeParameters, subbedType.arguments)
+                    if (itSubbed !is IrSimpleType) {
+                        logger.errorElement(
+                            "Substituted type has unexpected type ${itSubbed.javaClass}",
+                            callTarget
+                        )
+                        return listOf()
+                    }
+                    subbedType = itSubbed
+                }
+                return subbedType.arguments
+            }
+        }
+
+        private fun extractNewExprForLocalFunction(
+            ids: LocallyVisibleFunctionLabels,
+            parent: Label<out DbExprparent>,
+            locId: Label<DbLocation>,
+            enclosingCallable: Label<out DbCallable>,
+            enclosingStmt: Label<out DbStmt>
+        ) {
+
+            val idNewexpr =
+                extractNewExpr(
+                    ids.constructor,
+                    ids.type,
+                    locId,
+                    parent,
+                    -1,
+                    enclosingCallable,
+                    enclosingStmt
+                )
+            extractTypeAccessRecursive(
+                pluginContext.irBuiltIns.anyType,
+                locId,
+                idNewexpr,
+                -3,
+                enclosingCallable,
+                enclosingStmt
+            )
+        }
+
+        private fun extractMethodAccessWithoutArgs(
+            returnType: IrType,
+            locId: Label<DbLocation>,
+            enclosingCallable: Label<out DbCallable>,
+            callsiteParent: Label<out DbExprparent>,
+            childIdx: Int,
+            enclosingStmt: Label<out DbStmt>,
+            methodLabel: Label<out DbCallable>?
+        ) =
+            tw.getFreshIdLabel<DbMethodaccess>().also { id ->
+                val type = useType(returnType)
+
+                tw.writeExprs_methodaccess(id, type.javaResult.id, callsiteParent, childIdx)
+                tw.writeExprsKotlinType(id, type.kotlinResult.id)
+                extractExprContext(id, locId, enclosingCallable, enclosingStmt)
+
+                // The caller should have warned about this before, so we don't repeat the warning here.
+                if (methodLabel != null) tw.writeCallableBinding(id, methodLabel)
+            }
+
+        private val defaultConstructorMarkerClass by lazy {
+            referenceExternalClass("kotlin.jvm.internal.DefaultConstructorMarker")
+        }
+
+        private val defaultConstructorMarkerType by lazy { defaultConstructorMarkerClass?.typeWith() }
+
+        private fun getDefaultsMethodLastArgType(f: IrFunction) =
+            (if (f is IrConstructor) defaultConstructorMarkerType else null)
+                ?: pluginContext.irBuiltIns.anyType
+
+        private fun getDefaultsMethodArgTypes(f: IrFunction) =
+            // The $default method has type ([dispatchReceiver], [extensionReceiver], paramTypes...,
+            // int, Object)
+            // All parameter types are erased. The trailing int is a mask indicating which parameter
+            // values are real
+            // and which should be replaced by defaults. The final Object parameter is apparently always
+            // null.
+            (listOfNotNull(if (f.shouldExtractAsStatic) null else f.dispatchReceiverParameter?.type) +
+                    listOfNotNull(f.extensionReceiverParameter?.type) +
+                    f.valueParameters.map { it.type } +
+                    listOf(pluginContext.irBuiltIns.intType, getDefaultsMethodLastArgType(f)))
+                .map { erase(it) }
+
+        private fun getDefaultsMethodName(f: IrFunction) =
+            if (f is IrConstructor) {
+                f.returnType.let {
+                    when {
+                        it.isAnonymous -> ""
+                        else -> it.classFqName?.shortName()?.asString() ?: f.name.asString()
+                    }
+                }
+            } else {
+                getFunctionShortName(f).nameInDB + "\$default"
+            }
+
+        private fun getDefaultsMethodLabel(f: IrFunction): Label<out DbCallable>? {
+            val classTypeArgsIncludingOuterClasses = null
+            val parentId = useDeclarationParentOf(f, false, classTypeArgsIncludingOuterClasses, true)
+            if (parentId == null) {
+                logger.errorElement("Couldn't get parent ID for defaults method", f)
+                return null
+            }
+            return getDefaultsMethodLabel(f, parentId)
+        }
+
+        private fun getDefaultsMethodLabel(
+            f: IrFunction,
+            parentId: Label<out DbElement>
+        ): Label<out DbCallable> {
+            val defaultsMethodName = if (f is IrConstructor) "<init>" else getDefaultsMethodName(f)
+            val argTypes = getDefaultsMethodArgTypes(f)
+
+            val defaultMethodLabelStr =
+                getFunctionLabel(
+                    f.parent,
+                    parentId,
+                    defaultsMethodName,
+                    argTypes,
+                    erase(f.returnType),
+                    extensionParamType = null, // if there's any, that's included already in argTypes
+                    functionTypeParameters = listOf(),
+                    classTypeArgsIncludingOuterClasses = null,
+                    overridesCollectionsMethod = false,
+                    javaSignature = null,
+                    addParameterWildcardsByDefault = false
+                )
+
+            return tw.getLabelFor(defaultMethodLabelStr)
+        }
+
+        private fun extractsDefaultsCall(
+            syntacticCallTarget: IrFunction,
+            locId: Label<DbLocation>,
+            resultType: IrType,
+            enclosingCallable: Label<out DbCallable>,
+            callsiteParent: Label<out DbExprparent>,
+            childIdx: Int,
+            enclosingStmt: Label<out DbStmt>,
+            valueArguments: List<IrExpression?>,
+            dispatchReceiver: IrExpression?,
+            extensionReceiver: IrExpression?
+        ) {
+            val callTarget = syntacticCallTarget.target.realOverrideTarget
+            if (isExternalDeclaration(callTarget)) {
+                // Ensure the real target gets extracted, as we might not every directly touch it thanks
+                // to this call being redirected to a $default method.
+                useFunction<DbCallable>(callTarget)
+            }
+
+            // Default parameter values are inherited by overrides; in this case the call should
+            // dispatch against the $default method belonging to the class
+            // that specified the default values, which will in turn dynamically dispatch back to the
+            // relevant override.
+            val overriddenCallTarget =
+                (callTarget as? IrSimpleFunction)?.allOverriddenIncludingSelf()?.firstOrNull {
+                    it.overriddenSymbols.isEmpty() &&
+                        it.valueParameters.any { p -> p.defaultValue != null }
+                } ?: callTarget
+            if (isExternalDeclaration(overriddenCallTarget)) {
+                // Likewise, ensure the overridden target gets extracted.
+                useFunction<DbCallable>(overriddenCallTarget)
+            }
+
+            val defaultMethodLabel = getDefaultsMethodLabel(overriddenCallTarget)
+            val id =
+                extractMethodAccessWithoutArgs(
+                    resultType,
+                    locId,
+                    enclosingCallable,
+                    callsiteParent,
+                    childIdx,
+                    enclosingStmt,
+                    defaultMethodLabel
+                )
+
+            if (overriddenCallTarget.isLocalFunction()) {
+                extractTypeAccess(
+                    getLocallyVisibleFunctionLabels(overriddenCallTarget).type,
+                    locId,
+                    id,
+                    -1,
+                    enclosingCallable,
+                    enclosingStmt
+                )
+            } else {
+                extractStaticTypeAccessQualifierUnchecked(
+                    overriddenCallTarget,
+                    id,
+                    locId,
+                    enclosingCallable,
+                    enclosingStmt
+                )
+            }
+
+            extractDefaultsCallArguments(
+                id,
+                overriddenCallTarget,
+                enclosingCallable,
+                enclosingStmt,
+                valueArguments,
+                dispatchReceiver,
+                extensionReceiver
+            )
+        }
+
+        private fun extractDefaultsCallArguments(
+            id: Label<out DbExprparent>,
+            callTarget: IrFunction,
+            enclosingCallable: Label<out DbCallable>,
+            enclosingStmt: Label<out DbStmt>,
+            valueArguments: List<IrExpression?>,
+            dispatchReceiver: IrExpression?,
+            extensionReceiver: IrExpression?
+        ) {
+            var nextIdx = 0
+            if (dispatchReceiver != null && !callTarget.shouldExtractAsStatic) {
+                extractExpressionExpr(dispatchReceiver, enclosingCallable, id, nextIdx++, enclosingStmt)
+            }
+
+            if (extensionReceiver != null) {
+                extractExpressionExpr(
+                    extensionReceiver,
+                    enclosingCallable,
+                    id,
+                    nextIdx++,
+                    enclosingStmt
+                )
+            }
+
+            val valueArgsWithDummies =
+                valueArguments.zip(callTarget.valueParameters).map { (expr, param) ->
+                    expr ?: IrConstImpl.defaultValueForType(0, 0, param.type)
+                }
+
+            var realParamsMask = 0
+            valueArguments.forEachIndexed { index, arg ->
+                if (arg != null) realParamsMask = realParamsMask or (1 shl index)
+            }
+
+            val extraArgs =
+                listOf(
+                    IrConstImpl.int(0, 0, pluginContext.irBuiltIns.intType, realParamsMask),
+                    IrConstImpl.defaultValueForType(0, 0, getDefaultsMethodLastArgType(callTarget))
+                )
+
+            extractCallValueArguments(
+                id,
+                valueArgsWithDummies + extraArgs,
+                enclosingStmt,
+                enclosingCallable,
+                nextIdx,
+                extractVarargAsArray = true
+            )
+        }
+
+        private fun getFunctionInvokeMethod(typeArgs: List<IrTypeArgument>): IrFunction? {
+            // For `kotlin.FunctionX` and `kotlin.reflect.KFunctionX` interfaces, we're making sure that
+            // we
+            // extract the call to the `invoke` method that does exist,
+            // `kotlin.jvm.functions.FunctionX::invoke`.
+            val functionalInterface = getFunctionalInterfaceTypeWithTypeArgs(typeArgs)
+            if (functionalInterface == null) {
+                logger.warn("Cannot find functional interface type for raw method access")
+                return null
+            }
+            val functionalInterfaceClass = functionalInterface.classOrNull
+            if (functionalInterfaceClass == null) {
+                logger.warn("Cannot find functional interface class for raw method access")
+                return null
+            }
+            val interfaceType = functionalInterfaceClass.owner
+            val substituted = getJavaEquivalentClass(interfaceType) ?: interfaceType
+            val function = findFunction(substituted, OperatorNameConventions.INVOKE.asString())
+            if (function == null) {
+                logger.warn("Cannot find invoke function for raw method access")
+                return null
+            }
+            return function
+        }
+
+        private fun isFunctionInvoke(callTarget: IrFunction, drType: IrSimpleType) =
+            (drType.isFunctionOrKFunction() || drType.isSuspendFunctionOrKFunction()) &&
+                callTarget.name.asString() == OperatorNameConventions.INVOKE.asString()
+
+        private fun getCalleeMethodId(
+            callTarget: IrFunction,
+            drType: IrType?,
+            allowInstantiatedGenericMethod: Boolean
+        ): Label<out DbCallable>? {
+            if (callTarget.isLocalFunction())
+                return getLocallyVisibleFunctionLabels(callTarget).function
+
+            if (
+                allowInstantiatedGenericMethod &&
+                    drType is IrSimpleType &&
+                    !isUnspecialised(drType, logger)
+            ) {
+                val calleeIsInvoke = isFunctionInvoke(callTarget, drType)
+
+                val extractionMethod =
+                    if (calleeIsInvoke) getFunctionInvokeMethod(drType.arguments) else callTarget
+
+                return extractionMethod?.let {
+                    val typeArgs =
+                        if (calleeIsInvoke && drType.arguments.size > BuiltInFunctionArity.BIG_ARITY) {
+                            // Big arity `invoke` methods have a special implementation on JVM, they are
+                            // transformed to a call to
+                            // `kotlin.jvm.functions.FunctionN<out R>::invoke(vararg args: Any?)`, so we
+                            // only need to pass the type
+                            // argument for the return type. Additionally, the arguments are extracted
+                            // inside an array literal below.
+                            listOf(drType.arguments.last())
+                        } else {
+                            getDeclaringTypeArguments(callTarget, drType)
+                        }
+                    useFunction<DbCallable>(extractionMethod, typeArgs)
+                }
+            } else {
+                return useFunction<DbCallable>(callTarget)
+            }
+        }
+
+        private fun getCalleeRealOverrideTarget(f: IrFunction): IrFunction {
+            val target = f.target.realOverrideTarget
+            return if (overridesCollectionsMethodWithAlteredParameterTypes(f))
+            // Cope with the case where an inherited callee can be rewritten with substituted parameter
+            // types
+            // if the child class uses it to implement a collections interface
+            // (for example, `class A { boolean contains(Object o) { ... } }; class B<T> extends A
+            // implements Set<T> { ... }`
+            // leads to generating a function `A.contains(B::T)`, with `initialSignatureFunction`
+            // pointing to `A.contains(Object)`.
+            (target as? IrLazyFunction)?.initialSignatureFunction ?: target
+            else target
+        }
+
+        private fun callUsesDefaultArguments(
+            callTarget: IrFunction,
+            valueArguments: List<IrExpression?>
+        ): Boolean {
+            val varargParam = callTarget.valueParameters.withIndex().find { it.value.isVararg }
+            // If the vararg param is the only one not specified, and it has no default value, then we
+            // don't need to call a $default method,
+            // as omitting it already implies passing an empty vararg array.
+            val nullAllowedIdx =
+                if (varargParam != null && varargParam.value.defaultValue == null) varargParam.index
+                else -1
+            return valueArguments.withIndex().any { (index, it) ->
+                it == null && index != nullAllowedIdx
+            }
+        }
+
+        fun extractRawMethodAccess(
+            syntacticCallTarget: IrFunction,
+            locElement: IrElement,
+            resultType: IrType,
+            enclosingCallable: Label<out DbCallable>,
+            callsiteParent: Label<out DbExprparent>,
+            childIdx: Int,
+            enclosingStmt: Label<out DbStmt>,
+            valueArguments: List<IrExpression?>,
+            dispatchReceiver: IrExpression?,
+            extensionReceiver: IrExpression?,
+            typeArguments: List<IrType> = listOf(),
+            extractClassTypeArguments: Boolean = false,
+            superQualifierSymbol: IrClassSymbol? = null
+        ) {
+
+            val locId = tw.getLocation(locElement)
+
+            if (callUsesDefaultArguments(syntacticCallTarget, valueArguments)) {
+                extractsDefaultsCall(
+                    syntacticCallTarget,
+                    locId,
+                    resultType,
+                    enclosingCallable,
+                    callsiteParent,
+                    childIdx,
+                    enclosingStmt,
+                    valueArguments,
+                    dispatchReceiver,
+                    extensionReceiver
+                )
+            } else {
+                extractRawMethodAccess(
+                    syntacticCallTarget,
+                    locId,
+                    resultType,
+                    enclosingCallable,
+                    callsiteParent,
+                    childIdx,
+                    enclosingStmt,
+                    valueArguments.size,
+                    { argParent, idxOffset ->
+                        extractCallValueArguments(
+                            argParent,
+                            valueArguments,
+                            enclosingStmt,
+                            enclosingCallable,
+                            idxOffset
+                        )
+                    },
+                    dispatchReceiver?.type,
+                    dispatchReceiver?.let {
+                        { callId ->
+                            extractExpressionExpr(
+                                dispatchReceiver,
+                                enclosingCallable,
+                                callId,
+                                -1,
+                                enclosingStmt
+                            )
+                        }
+                    },
+                    extensionReceiver?.let {
+                        { argParent ->
+                            extractExpressionExpr(
+                                extensionReceiver,
+                                enclosingCallable,
+                                argParent,
+                                0,
+                                enclosingStmt
+                            )
+                        }
+                    },
+                    typeArguments,
+                    extractClassTypeArguments,
+                    superQualifierSymbol
+                )
+            }
+        }
+
+        private fun extractStaticTypeAccessQualifierUnchecked(
+            target: IrDeclaration,
+            parentExpr: Label<out DbExprparent>,
+            locId: Label<DbLocation>,
+            enclosingCallable: Label<out DbCallable>?,
+            enclosingStmt: Label<out DbStmt>?
+        ) {
+            val parent = target.parent
+            if (parent is IrExternalPackageFragment) {
+                // This is in a file class.
+                val fqName = getFileClassFqName(target)
+                if (fqName == null) {
+                    logger.error(
+                        "Can't get FqName for static type access qualifier in external package fragment ${target.javaClass}"
+                    )
+                } else {
+                    extractTypeAccess(
+                        useFileClassType(fqName),
+                        locId,
+                        parentExpr,
+                        -1,
+                        enclosingCallable,
+                        enclosingStmt
+                    )
+                }
+            } else if (parent is IrClass) {
+                extractTypeAccessRecursive(
+                    parent.toRawType(),
+                    locId,
+                    parentExpr,
+                    -1,
+                    enclosingCallable,
+                    enclosingStmt
+                )
+            } else if (parent is IrFile) {
+                extractTypeAccess(
+                    useFileClassType(parent),
+                    locId,
+                    parentExpr,
+                    -1,
+                    enclosingCallable,
+                    enclosingStmt
+                )
+            } else {
+                logger.warnElement(
+                    "Unexpected static type access qualifier ${parent.javaClass}",
+                    parent
+                )
+            }
+        }
+
+        private fun extractStaticTypeAccessQualifier(
+            target: IrDeclaration,
+            parentExpr: Label<out DbExprparent>,
+            locId: Label<DbLocation>,
+            enclosingCallable: Label<out DbCallable>?,
+            enclosingStmt: Label<out DbStmt>?
+        ) {
+            if (target.shouldExtractAsStatic) {
+                extractStaticTypeAccessQualifierUnchecked(
+                    target,
+                    parentExpr,
+                    locId,
+                    enclosingCallable,
+                    enclosingStmt
+                )
+            }
+        }
+
+        private fun isStaticAnnotatedNonCompanionMember(f: IrSimpleFunction) =
+            f.parentClassOrNull?.isNonCompanionObject == true &&
+                (f.hasAnnotation(jvmStaticFqName) ||
+                    f.correspondingPropertySymbol?.owner?.hasAnnotation(jvmStaticFqName) == true)
+
+        private val IrDeclaration.shouldExtractAsStatic: Boolean
+            get() =
+                this is IrSimpleFunction &&
+                    (isStaticFunction(this) || isStaticAnnotatedNonCompanionMember(this)) ||
+                    this is IrField && this.isStatic ||
+                    this is IrEnumEntry
+
+        private fun extractCallValueArguments(
+            callId: Label<out DbExprparent>,
+            call: IrFunctionAccessExpression,
+            enclosingStmt: Label<out DbStmt>,
+            enclosingCallable: Label<out DbCallable>,
+            idxOffset: Int
+        ) =
+            extractCallValueArguments(
+                callId,
+                (0 until call.valueArgumentsCount).map { call.getValueArgument(it) },
+                enclosingStmt,
+                enclosingCallable,
+                idxOffset
+            )
+
+
+
+ */

java/kotlin-extractor2/src/main/kotlin/entities/TypeArgument.kt

@@ -0,0 +1,69 @@
+package com.github.codeql.entities
+
+import com.github.codeql.*
+import org.jetbrains.kotlin.analysis.api.KaSession
+import org.jetbrains.kotlin.analysis.api.types.KaStarTypeProjection
+import org.jetbrains.kotlin.analysis.api.types.KaTypeArgumentWithVariance
+import org.jetbrains.kotlin.analysis.api.types.KaTypeProjection
+import org.jetbrains.kotlin.types.Variance
+
+context(KaSession)
+fun KotlinUsesExtractor.getTypeArgumentLabel(arg: KaTypeProjection): TypeResultWithoutSignature<DbReftype> {
+
+    fun extractBoundedWildcard(
+        wildcardKind: Int,
+        wildcardLabelStr: String,
+        wildcardShortName: String,
+        boundLabel: Label<out DbReftype>
+    ): Label<DbWildcard> =
+        tw.getLabelFor(wildcardLabelStr) { wildcardLabel ->
+            tw.writeWildcards(wildcardLabel, wildcardShortName, wildcardKind)
+            tw.writeHasLocation(wildcardLabel, tw.unknownLocation)
+            tw.getLabelFor<DbTypebound>("@\"bound;0;{$wildcardLabel}\"") {
+                tw.writeTypeBounds(it, boundLabel, 0, wildcardLabel)
+            }
+        }
+
+    // Note this function doesn't return a signature because type arguments are never
+    // incorporated into function signatures.
+    return when (arg) {
+        is KaStarTypeProjection -> {
+            val anyTypeLabel =
+                useType(builtinTypes.any).javaResult.id.cast<DbReftype>()
+            TypeResultWithoutSignature(
+                extractBoundedWildcard(1, "@\"wildcard;\"", "?", anyTypeLabel),
+                /* OLD: KE1
+                Unit, */
+                "?"
+            )
+        }
+        is KaTypeArgumentWithVariance -> {
+            val boundResults = useType(arg.type, TypeContext.GENERIC_ARGUMENT)
+            val boundLabel = boundResults.javaResult.id.cast<DbReftype>()
+
+            if (arg.variance == Variance.INVARIANT)
+                boundResults.javaResult.cast<DbReftype>().forgetSignature()
+            else {
+                val keyPrefix = if (arg.variance == Variance.IN_VARIANCE) "super" else "extends"
+                val wildcardKind = if (arg.variance == Variance.IN_VARIANCE) 2 else 1
+                val wildcardShortName = "? $keyPrefix ${boundResults.javaResult.shortName}"
+                TypeResultWithoutSignature(
+                    extractBoundedWildcard(
+                        wildcardKind,
+                        "@\"wildcard;$keyPrefix{$boundLabel}\"",
+                        wildcardShortName,
+                        boundLabel
+                    ),
+                    /* OLD: KE1
+                    Unit,
+                    */
+                    wildcardShortName
+                )
+            }
+        }
+        else -> {
+            logger.error("Unexpected type argument: " + arg.javaClass)
+            extractJavaErrorType().forgetSignature()
+        }
+    }
+}

java/kotlin-extractor2/src/main/kotlin/entities/TypeParameter.kt

@@ -0,0 +1,90 @@
+package com.github.codeql.entities
+
+import com.github.codeql.*
+import com.intellij.psi.PsiElement
+import org.jetbrains.kotlin.analysis.api.KaSession
+import org.jetbrains.kotlin.analysis.api.symbols.KaTypeParameterSymbol
+import org.jetbrains.kotlin.analysis.api.symbols.psiSafe
+import org.jetbrains.kotlin.analysis.api.types.KaTypeParameterType
+import org.jetbrains.kotlin.types.Variance
+
+context(KaSession)
+fun KotlinUsesExtractor.getTypeParameterLabel(param: KaTypeParameterSymbol, parentLabel: Label<out DbClassorinterfaceorcallable>): String {
+    return "@\"typevar;{$parentLabel};${param.name}\""
+}
+
+context(KaSession)
+fun KotlinUsesExtractor.getTypeParameterLabel(param: KaTypeParameterSymbol) =
+    // Use this instead of `useDeclarationParent` so we can use useFunction with noReplace = true,
+    // ensuring that e.g. a method-scoped type variable declared on kotlin.String.transform<R> gets
+    // a different name to the corresponding java.lang.String.transform<R>, even though
+    // useFunction will usually replace references to one function with the other.
+    getTypeParameterParentLabel(param)?.let {
+        getTypeParameterLabel(param, it)
+    }
+
+context(KaSession)
+fun KotlinFileExtractor.extractTypeParameter(
+    tp: KaTypeParameterSymbol,
+    apparentIndex: Int,
+    /* OLD: KE1
+    javaTypeParameter: JavaTypeParameter? */
+): Label<out DbTypevariable>? {
+    with("type parameter", tp) {
+        val parentId = getTypeParameterParentLabel(tp) ?: return null
+        val id = tw.getLabelFor<DbTypevariable>(getTypeParameterLabel(tp, parentId))
+
+        /* COMMENT OLD: KE1 -- check if this still applies */
+        // Note apparentIndex does not necessarily equal `tp.index`, because at least constructor type parameters
+        // have indices offset from the type parameters of the constructed class (i.e. the parameter S of
+        // `class Generic<T> { public <S> Generic(T t, S s) { ... } }` will have `tp.index` 1, not 0).
+        tw.writeTypeVars(id, tp.name.asString(), apparentIndex, parentId)
+        val locId = tp.psiSafe<PsiElement>()?.let { tw.getLocation(it) } ?: tw.getWholeFileLocation()
+        tw.writeHasLocation(id, locId)
+
+        /* OLD: KE1
+        // Annoyingly, we have no obvious way to pair up the bounds of an IrTypeParameter and a  JavaTypeParameter
+        // because JavaTypeParameter provides a Collection not an ordered list, so we can only do our best here:
+        fun tryGetJavaBound(idx: Int) =
+            when (tp.superTypes.size) {
+                1 -> javaTypeParameter?.upperBounds?.singleOrNull()
+                else -> (javaTypeParameter?.upperBounds as? List)?.getOrNull(idx)
+            }
+         */
+
+        tp.upperBounds.forEachIndexed { boundIdx, bound ->
+            if (!bound.upperBoundIfFlexible().isAnyType) {
+                tw.getLabelFor<DbTypebound>("@\"bound;$boundIdx;{$id}\"") {
+                    /* COMMENT OLD: KE1 -- check if this still applies */
+                    // Note we don't look for @JvmSuppressWildcards here because it doesn't seem to have any impact
+                    // on kotlinc adding wildcards to type parameter bounds.
+                    val boundWithWildcards = bound
+                    /* OLD: KE1
+                        addJavaLoweringWildcards(bound, true, tryGetJavaBound(tp.index))
+                     */
+                    tw.writeTypeBounds(
+                        it,
+                        useType(boundWithWildcards).javaResult.id.cast<DbReftype>(),
+                        boundIdx,
+                        id
+                    )
+                }
+            }
+        }
+
+        if (tp.isReified) {
+            addModifiers(id, "reified")
+        }
+
+        when (tp.variance) {
+            Variance.IN_VARIANCE -> addModifiers(id, "in")
+            Variance.OUT_VARIANCE -> addModifiers(id, "out")
+            else -> {}
+        }
+
+        // extractAnnotations(tp, id)
+        // TODO: introduce annotations once they can be disambiguated from bounds, which are
+        // also child expressions.
+        return id
+    }
+}

java/kotlin-extractor2/src/main/kotlin/entities/Types.kt

@@ -0,0 +1,327 @@
+package com.github.codeql
+
+import com.github.codeql.entities.getTypeParameterLabel
+import org.jetbrains.kotlin.analysis.api.KaSession
+import org.jetbrains.kotlin.analysis.api.components.DefaultTypeClassIds
+import org.jetbrains.kotlin.analysis.api.symbols.KaClassSymbol
+import org.jetbrains.kotlin.analysis.api.symbols.KaFunctionSymbol
+import org.jetbrains.kotlin.analysis.api.symbols.KaTypeParameterSymbol
+import org.jetbrains.kotlin.analysis.api.types.*
+
+context(KaSession)
+private fun KotlinUsesExtractor.useClassType(
+    c: KaClassType
+): TypeResults {
+    // TODO: this cast is unsafe; .symbol is actually a KaClassLikeSymbol
+    val javaResult = addClassLabel(c.symbol as KaClassSymbol, c.qualifiers.map { it.typeArguments })
+    // TODO: Actually the Kotlin class ID should sometimes differ from javaResult.id, e.g.
+    //   when collections types differ.
+    val kotlinTypeId =
+        tw.getLabelFor<DbKt_class_type>("@\"kt_class;{${javaResult.id}}\"") {
+            tw.writeKt_class_types(it, javaResult.id)
+        }
+    val kotlinResult = TypeResult(kotlinTypeId /* , "TODO"*/, "TODO")
+    return TypeResults(javaResult, kotlinResult)
+}
+
+context(KaSession)
+fun KotlinUsesExtractor.getTypeParameterParentLabel(param: KaTypeParameterSymbol) =
+    param.containingSymbol?.let {
+        when (it) {
+            is KaClassSymbol -> useClassSource(it)
+            is KaFunctionSymbol ->
+                /* OLD: KE1
+                (if (this is KotlinFileExtractor)
+                    this.declarationStack
+                        .findOverriddenAttributes(it)
+                        ?.takeUnless {
+                            // When extracting the `static fun f$default(...)` that accompanies
+                            // `fun <T> f(val x: T? = defaultExpr, ...)`,
+                            // `f$default` has no type parameters, and so there is no
+                            // `f$default::T` to refer to.
+                            // We have no good way to extract references to `T` in
+                            // `defaultExpr`, so we just fall back on describing it
+                            // in terms of `f::T`, even though that type variable ought to be
+                            // out of scope here.
+                                attribs ->
+                            attribs.typeParameters?.isEmpty() == true
+                        }
+                        ?.id
+                else null) ?:
+                */
+                useFunction(it, useDeclarationParentOf(it, true) ?: TODO(), listOf(),  /* OLD: KE1 noReplace = true */)
+            else -> {
+                logger.error("Unexpected type parameter parent $it")
+                null
+            }
+        }
+    }
+
+context(KaSession)
+private fun KotlinUsesExtractor.useTypeParameterType(param: KaTypeParameterType) =
+    getTypeParameterLabel(param.symbol)?.let {
+        TypeResult(
+            tw.getLabelFor<DbTypevariable>(it),
+            /* OLD: KE1
+            useType(eraseTypeParameter(param)).javaResult.signature,
+             */
+            param.name.asString()
+        )
+    } ?: extractErrorType()
+
+context(KaSession)
+fun KotlinUsesExtractor.useType(t: KaType?, context: TypeContext = TypeContext.OTHER): TypeResults {
+    val tr = when (t) {
+        null -> {
+            logger.error("Unexpected null type")
+            return extractErrorType()
+        }
+        is KaClassType -> useClassType(t)
+        is KaFlexibleType -> useType(t.lowerBound) // TODO: take a more reasoned choice here
+        is KaTypeParameterType -> TypeResults(useTypeParameterType(t), extractErrorType().kotlinResult /* TODO */)
+        else -> TODO()
+    }
+    val javaResult = tr.javaResult
+    val kotlinResultBase = tr.kotlinResult
+    val abbreviation = t.abbreviatedType
+    val kotlinResultAlias = if (abbreviation == null) kotlinResultBase else {
+                                // TODO: this cast is unsafe; .symbol is actually a KaClassLikeSymbol
+                                val classResult = addClassLabel(abbreviation.symbol as KaClassSymbol, listOf<Nothing>() /* TODO */)
+                                val kotlinBaseTypeId = kotlinResultBase.id
+                                val kotlinAliasTypeId =
+                                    tw.getLabelFor<DbKt_type_alias>("@\"kt_type_alias;{${classResult.id}};{$kotlinBaseTypeId}\"") {
+                                        tw.writeKt_type_aliases(it, classResult.id, kotlinBaseTypeId)
+                                    }
+                                TypeResult(kotlinAliasTypeId , "TODO"/*, "TODO" */)
+                            }
+    val kotlinResultNullability = if (t.nullability.isNullable) {
+                                      val kotlinAliasTypeId = kotlinResultAlias.id
+                                      val kotlinNullableTypeId =
+                                          tw.getLabelFor<DbKt_nullable_type>("@\"kt_nullable_type;{$kotlinAliasTypeId}\"") {
+                                              tw.writeKt_nullable_types(it, kotlinAliasTypeId)
+                                          }
+                                      TypeResult(kotlinNullableTypeId, "TODO", /* "TODO" */)
+                                  } else kotlinResultAlias
+    return TypeResults(javaResult, kotlinResultNullability)
+}
+
+fun KotlinUsesExtractor.extractJavaErrorType(): TypeResult<DbErrortype> {
+    val typeId = tw.getLabelFor<DbErrortype>("@\"errorType\"") { tw.writeError_type(it) }
+    return TypeResult(typeId, /* TODO , */ "<CodeQL error type>")
+}
+
+private fun KotlinUsesExtractor.extractErrorType(): TypeResults {
+    val javaResult = extractJavaErrorType()
+    val kotlinTypeId =
+        tw.getLabelFor<DbKt_error_type>("@\"errorKotlinType\"") {
+            tw.writeKt_error_types(it)
+        }
+    return TypeResults(
+        javaResult,
+        TypeResult(kotlinTypeId, /* TODO, */"<CodeQL error type>")
+    )
+}
+
+/*
+OLD: KE1
+    // `args` can be null to describe a raw generic type.
+    // For non-generic types it will be zero-length list.
+    fun useSimpleTypeClass(
+        c: IrClass,
+        args: List<IrTypeArgument>?,
+        hasQuestionMark: Boolean
+    ): TypeResults {
+        val classInstanceResult = useClassInstance(c, args)
+        val javaClassId = classInstanceResult.typeResult.id
+        val kotlinQualClassName = getUnquotedClassLabel(c, args).classLabel
+        val javaResult = classInstanceResult.typeResult
+        val kotlinResult =
+            if (true) TypeResult(fakeKotlinType(), "TODO", "TODO")
+            else if (hasQuestionMark) {
+                val kotlinSignature = "$kotlinQualClassName?" // TODO: Is this right?
+                val kotlinLabel = "@\"kt_type;nullable;$kotlinQualClassName\""
+                val kotlinId: Label<DbKt_nullable_type> =
+                    tw.getLabelFor(kotlinLabel, { tw.writeKt_nullable_types(it, javaClassId) })
+                TypeResult(kotlinId, kotlinSignature, "TODO")
+            } else {
+                val kotlinSignature = kotlinQualClassName // TODO: Is this right?
+                val kotlinLabel = "@\"kt_type;notnull;$kotlinQualClassName\""
+                val kotlinId: Label<DbKt_notnull_type> =
+                    tw.getLabelFor(kotlinLabel, { tw.writeKt_notnull_types(it, javaClassId) })
+                TypeResult(kotlinId, kotlinSignature, "TODO")
+            }
+        return TypeResults(javaResult, kotlinResult)
+    }
+*/
+
+enum class TypeContext {
+    RETURN,
+    GENERIC_ARGUMENT,
+    OTHER
+}
+
+/*
+OLD: KE1
+    private fun useSimpleType(s: IrSimpleType, context: TypeContext): TypeResults {
+        if (s.abbreviation != null) {
+            // TODO: Extract this information
+        }
+        // We use this when we don't actually have an IrClass for a class
+        // we want to refer to
+        // TODO: Eliminate the need for this if possible
+        fun makeClass(pkgName: String, className: String): Label<DbClassorinterface> {
+            val pkgId = extractPackage(pkgName)
+            val label = "@\"class;$pkgName.$className\""
+            val classId: Label<DbClassorinterface> =
+                tw.getLabelFor(label, { tw.writeClasses_or_interfaces(it, className, pkgId, it) })
+            return classId
+        }
+        fun primitiveType(
+            kotlinClass: IrClass,
+            primitiveName: String?,
+            otherIsPrimitive: Boolean,
+            javaClass: IrClass,
+            kotlinPackageName: String,
+            kotlinClassName: String
+        ): TypeResults {
+            // Note the use of `hasEnhancedNullability` here covers cases like `@NotNull Integer`,
+            // which must be extracted as `Integer` not `int`.
+            val javaResult =
+                if (
+                    (context == TypeContext.RETURN ||
+                        (context == TypeContext.OTHER && otherIsPrimitive)) &&
+                        !s.isNullable() &&
+                        getKotlinType(s)?.hasEnhancedNullability() != true &&
+                        primitiveName != null
+                ) {
+                    val label: Label<DbPrimitive> =
+                        tw.getLabelFor(
+                            "@\"type;$primitiveName\"",
+                            { tw.writePrimitives(it, primitiveName) }
+                        )
+                    TypeResult(label, primitiveName, primitiveName)
+                } else {
+                    addClassLabel(javaClass, listOf())
+                }
+            val kotlinClassId = useClassInstance(kotlinClass, listOf()).typeResult.id
+            val kotlinResult =
+                if (true) TypeResult(fakeKotlinType(), "TODO", "TODO")
+                else if (s.isNullable()) {
+                    val kotlinSignature =
+                        "$kotlinPackageName.$kotlinClassName?" // TODO: Is this right?
+                    val kotlinLabel = "@\"kt_type;nullable;$kotlinPackageName.$kotlinClassName\""
+                    val kotlinId: Label<DbKt_nullable_type> =
+                        tw.getLabelFor(
+                            kotlinLabel,
+                            { tw.writeKt_nullable_types(it, kotlinClassId) }
+                        )
+                    TypeResult(kotlinId, kotlinSignature, "TODO")
+                } else {
+                    val kotlinSignature =
+                        "$kotlinPackageName.$kotlinClassName" // TODO: Is this right?
+                    val kotlinLabel = "@\"kt_type;notnull;$kotlinPackageName.$kotlinClassName\""
+                    val kotlinId: Label<DbKt_notnull_type> =
+                        tw.getLabelFor(kotlinLabel, { tw.writeKt_notnull_types(it, kotlinClassId) })
+                    TypeResult(kotlinId, kotlinSignature, "TODO")
+                }
+            return TypeResults(javaResult, kotlinResult)
+        }
+
+        val owner = s.classifier.owner
+        val primitiveInfo = primitiveTypeMapping.getPrimitiveInfo(s)
+
+        when {
+            primitiveInfo != null -> {
+                if (owner is IrClass) {
+                    return primitiveType(
+                        owner,
+                        primitiveInfo.primitiveName,
+                        primitiveInfo.otherIsPrimitive,
+                        primitiveInfo.javaClass,
+                        primitiveInfo.kotlinPackageName,
+                        primitiveInfo.kotlinClassName
+                    )
+                } else {
+                    logger.error(
+                        "Got primitive info for non-class (${owner.javaClass}) for ${s.render()}"
+                    )
+                    return extractErrorType()
+                }
+            }
+            (s.isBoxedArray && s.arguments.isNotEmpty()) || s.isPrimitiveArray() -> {
+                val arrayInfo = useArrayType(s, false)
+                return arrayInfo.componentTypeResults
+            }
+            owner is IrClass -> {
+                val args = if (s.isRawType()) null else s.arguments
+
+                return useSimpleTypeClass(owner, args, s.isNullable())
+            }
+            owner is IrTypeParameter -> {
+                val javaResult = useTypeParameter(owner)
+                val aClassId = makeClass("kotlin", "TypeParam") // TODO: Wrong
+                val kotlinResult =
+                    if (true) TypeResult(fakeKotlinType(), "TODO", "TODO")
+                    else if (s.isNullable()) {
+                        val kotlinSignature = "${javaResult.signature}?" // TODO: Wrong
+                        val kotlinLabel = "@\"kt_type;nullable;type_param\"" // TODO: Wrong
+                        val kotlinId: Label<DbKt_nullable_type> =
+                            tw.getLabelFor(kotlinLabel, { tw.writeKt_nullable_types(it, aClassId) })
+                        TypeResult(kotlinId, kotlinSignature, "TODO")
+                    } else {
+                        val kotlinSignature = javaResult.signature // TODO: Wrong
+                        val kotlinLabel = "@\"kt_type;notnull;type_param\"" // TODO: Wrong
+                        val kotlinId: Label<DbKt_notnull_type> =
+                            tw.getLabelFor(kotlinLabel, { tw.writeKt_notnull_types(it, aClassId) })
+                        TypeResult(kotlinId, kotlinSignature, "TODO")
+                    }
+                return TypeResults(javaResult, kotlinResult)
+            }
+            else -> {
+                logger.error("Unrecognised IrSimpleType: " + s.javaClass + ": " + s.render())
+                return extractErrorType()
+            }
+        }
+    }
+*/
+
+/**
+ * Returns `t` with generic types replaced by raw types, and type parameters replaced by their first bound.
+ *
+ * Note that `Array<T>` is retained (with `T` itself erased) because these are expected to be lowered to Java arrays,
+ * which are not generic.
+ */
+context(KaSession)
+fun erase(t: KaType): KaType =
+    when (t) {
+        is KaTypeParameterType -> erase(t.symbol.upperBounds.getOrElse(0) { buildClassType(DefaultTypeClassIds.ANY) })
+        is KaClassType -> buildClassType(t.classId)
+        is KaFlexibleType -> erase(t.lowerBound) // TODO: Check this -- see also useType's treatment of flexible types.
+        else -> t
+    }
+/* OLD: KE1 -- note might need to restore creation of a RAW type
+    when (val sym = t.symbol) {
+        is KaTypeParameterSymbol -> erase(sym.upperBounds[0])
+        is KaClassSymbol -> {
+            buildClassType()
+        }
+
+        if (owner is IrClass) {
+            if (t.isBoxedArray) {
+                val elementType = t.getArrayElementType(pluginContext.irBuiltIns)
+                val erasedElementType = erase(elementType)
+                return owner
+                    .typeWith(erasedElementType)
+                    .codeQlWithHasQuestionMark(t.isNullable())
+            }
+
+            return if (t.arguments.isNotEmpty())
+                t.addAnnotations(listOf(RawTypeAnnotation.annotationConstructor))
+            else t
+        }
+    }
+    return t
+}
+
+private fun eraseTypeParameter(t: KaTypeParameterSymbol) = erase(t.upperBounds[0])
+*/

java/kotlin-extractor2/src/main/kotlin/utils/AutoCloseableUse.kt

@@ -0,0 +1,47 @@
+package com.github.codeql
+
+// Functions copied from stdlib/jdk7/src/kotlin/AutoCloseable.kt, which is not available within
+// kotlinc,
+// but allows the `.use` pattern to be applied to JDK7 AutoCloseables:
+
+/**
+ * Executes the given [block] function on this resource and then closes it down correctly whether an
+ * exception is thrown or not.
+ *
+ * In case if the resource is being closed due to an exception occurred in [block], and the closing
+ * also fails with an exception, the latter is added to the
+ * [suppressed][java.lang.Throwable.addSuppressed] exceptions of the former.
+ *
+ * @param block a function to process this [AutoCloseable] resource.
+ * @return the result of [block] function invoked on this resource.
+ */
+public inline fun <T : AutoCloseable?, R> T.useAC(block: (T) -> R): R {
+    var exception: Throwable? = null
+    try {
+        return block(this)
+    } catch (e: Throwable) {
+        exception = e
+        throw e
+    } finally {
+        this.closeFinallyAC(exception)
+    }
+}
+
+/**
+ * Closes this [AutoCloseable], suppressing possible exception or error thrown by
+ * [AutoCloseable.close] function when it's being closed due to some other [cause] exception
+ * occurred.
+ *
+ * The suppressed exception is added to the list of suppressed exceptions of [cause] exception.
+ */
+fun AutoCloseable?.closeFinallyAC(cause: Throwable?) =
+    when {
+        this == null -> {}
+        cause == null -> close()
+        else ->
+            try {
+                close()
+            } catch (closeException: Throwable) {
+                cause.addSuppressed(closeException)
+            }
+    }

java/kotlin-extractor2/src/main/kotlin/utils/ClassNames.kt

@@ -0,0 +1,170 @@
+package com.github.codeql
+
+import com.intellij.openapi.vfs.StandardFileSystems
+import com.intellij.openapi.vfs.VirtualFile
+import org.jetbrains.kotlin.analysis.api.symbols.KaClassSymbol
+import org.jetbrains.kotlin.analysis.api.symbols.KaFileSymbol
+import org.jetbrains.kotlin.analysis.api.symbols.KaNamedClassSymbol
+import org.jetbrains.kotlin.analysis.api.symbols.KaSymbol
+import org.jetbrains.kotlin.analysis.api.symbols.markers.KaNamedSymbol
+import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.psi.*
+
+/*
+OLD: KE1
+import com.github.codeql.utils.getJvmName
+import com.github.codeql.utils.versions.*
+import com.intellij.openapi.vfs.StandardFileSystems
+import com.intellij.openapi.vfs.VirtualFile
+import org.jetbrains.kotlin.builtins.jvm.JavaToKotlinClassMap
+import org.jetbrains.kotlin.fir.java.JavaBinarySourceElement
+import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.declarations.*
+import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable
+import org.jetbrains.kotlin.ir.util.parentClassOrNull
+import org.jetbrains.kotlin.load.java.sources.JavaSourceElement
+import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
+import org.jetbrains.kotlin.load.kotlin.JvmPackagePartSource
+import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinarySourceElement
+import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClass
+*/
+
+// Adapted from Kotlin's interpreter/Utils.kt function 'internalName'
+// Translates class names into their JLS section 13.1 binary name,
+// and declarations within them into the parent class' JLS 13.1 name as
+// specified above, followed by a `$` separator and then the short name
+// for `that`.
+private fun getName(d: KaNamedSymbol) =
+    d.name.identifier
+    /* OLD: KE1
+    (d as? IrAnnotationContainer)?.let { getJvmName(it) } ?: d.name.asString()
+    */
+
+fun getFileClassName(f: KtFile): String =
+    null /* OLD: KE1: getJvmName(f) */
+        ?: ((f.virtualFilePath
+            .replaceFirst(Regex(""".*[/\\]"""), "")
+            .replaceFirst(Regex("""\.kt$"""), "")
+            .replaceFirstChar { it.uppercase() }) + "Kt")
+
+private fun getBinaryName(cid: ClassId): String =
+    (cid.outerClassId?.let { ocid -> "${getBinaryName(ocid)}\$" } ?: "${cid.packageFqName}.") + cid.shortClassName
+
+fun getSymbolBinaryName(that: KaSymbol): String {
+    if (that is KaFileSymbol) {
+        return "TODO"
+        /* OLD: KE1
+        val shortName = getFileClassName(that)
+        val pkg = that.packageFqName.asString()
+        return if (pkg.isEmpty()) shortName else "$pkg.$shortName"
+        */
+    }
+
+    /* OLD: KE1
+    if (that !is IrDeclaration) {
+        return "(unknown-name)"
+    }
+    */
+
+    val internalName =
+        (that as? KaNamedClassSymbol)?.classId?.let { getBinaryName(it) }
+            ?: "(unknown-binary-name)"
+    /* OLD: KE1
+    if (that !is KaClassSymbol) {
+
+        val parent = that.parent
+        if (parent is IrFile) {
+            // Note we'll fall through and do the IrPackageFragment case as well, since IrFile <:
+            // IrPackageFragment
+            internalName.insert(0, getFileClassName(parent) + "$")
+        }
+    }
+    */
+
+    return internalName
+}
+
+fun getIrClassVirtualFile(c: KaClassSymbol): VirtualFile? {
+    return c.psi?.containingFile?.virtualFile
+    /* OLD: KE1
+    val cSource = irClass.source
+    // Don't emit a location for multi-file classes until we're sure we can cope with different
+    // declarations
+    // inside a class disagreeing about their source file. In particular this currently causes
+    // problems when
+    // a source-location for a declarations tries to refer to a file-id which is assumed to be
+    // declared in
+    // the class trap file.
+    if (irClass.origin == IrDeclarationOrigin.JVM_MULTIFILE_CLASS) return null
+    when (cSource) {
+        is JavaSourceElement -> {
+            val element = cSource.javaElement
+            when (element) {
+                is BinaryJavaClass -> return element.virtualFile
+            }
+        }
+        is JavaBinarySourceElement -> {
+            return cSource.javaClass.virtualFile
+        }
+        is KotlinJvmBinarySourceElement -> {
+            val binaryClass = cSource.binaryClass
+            when (binaryClass) {
+                is VirtualFileKotlinClass -> return binaryClass.file
+            }
+        }
+        is JvmPackagePartSource -> {
+            val binaryClass = cSource.knownJvmBinaryClass
+            if (binaryClass != null && binaryClass is VirtualFileKotlinClass) {
+                return binaryClass.file
+            }
+        }
+    }
+    return null
+    */
+}
+
+private fun getRawClassSymbolBinaryPath(c: KaClassSymbol) =
+    getIrClassVirtualFile(c)?.let {
+        val path = it.path
+        if (it.fileSystem.protocol == StandardFileSystems.JRT_PROTOCOL)
+        // For JRT files, which we assume to be the JDK, hide the containing JAR path to match the
+        // Java extractor's behaviour.
+        "/${path.split("!/", limit = 2)[1]}"
+        else path
+    }
+
+fun getClassSymbolBinaryPath(c: KaClassSymbol): String {
+    return getRawClassSymbolBinaryPath(c)
+        // Otherwise, make up a fake location:
+        ?: getUnknownBinaryLocation(getSymbolBinaryName(c))
+}
+
+fun getSymbolBinaryPath(d: KaSymbol): String? {
+    if (d is KaClassSymbol) {
+        return getClassSymbolBinaryPath(d)
+    }
+    /*
+    OLD: KE1
+    val parentClass = d.parentClassOrNull
+    if (parentClass != null) {
+        return getIrClassBinaryPath(parentClass)
+    }
+    if (d.parent is IrExternalPackageFragment) {
+        // This is in a file class.
+        val fqName = getFileClassFqName(d)
+        if (fqName != null) {
+            return getUnknownBinaryLocation(fqName.asString())
+        }
+    }
+    */
+    return null
+}
+
+private fun getUnknownBinaryLocation(s: String): String {
+    return "/!unknown-binary-location/${s.replace(".", "/")}.class"
+}
+
+/* OLD: KE1
+fun getJavaEquivalentClassId(c: IrClass) =
+    c.fqNameWhenAvailable?.toUnsafe()?.let { JavaToKotlinClassMap.mapKotlinToJava(it) }
+*/