KE2: Start actually emitting some TRAP

java/kotlin-extractor2/BUILD.bazel

@@ -13,23 +13,23 @@ kt_kotlinc_options(
     # if needed, see https://bazelbuild.github.io/rules_kotlin/kotlin.html#kt_kotlinc_options for available options
 )
 
-# uncomment if you need the DB scheme generated bindings
-#py_binary(
-#    name = "generate_dbscheme",
-#    srcs = ["generate_dbscheme.py"],
-#)
-#genrule(
-#    name = "generated-dbscheme",
-#    srcs = ["@codeql//java:dbscheme"],
-#    outs = ["KotlinExtractorDbScheme.kt"],
-#    cmd = "$(execpath :generate_dbscheme) $< $@",
-#    tools = [":generate_dbscheme"],
-#)
+py_binary(
+    name = "generate_dbscheme",
+    srcs = ["generate_dbscheme.py"],
+)
+genrule(
+    name = "generated-dbscheme",
+    srcs = ["@codeql//java:dbscheme"],
+    outs = ["KotlinExtractorDbScheme.kt"],
+    cmd = "$(execpath :generate_dbscheme) $< $@",
+    tools = [":generate_dbscheme"],
+)
 
 kt_jvm_library(
     name = "ke2-kt",
     srcs = [
-        #        ":generated-dbscheme",
+        ":generated-dbscheme",
+        "src/main/java/com/semmle/util/unicode/UTF8Util.java",
     ] + glob(["src/main/kotlin/**/*.kt"]),
     javac_opts = ":javac-options",
     kotlinc_opts = ":kotlinc-options",

java/kotlin-extractor2/ke2.sh

@@ -14,6 +14,7 @@ mkdir -p "$TRAP_DIR"
 INVOCATION_TRAP=`mktemp -p "$TRAP_DIR" invocation.XXXXXXXXXX.trap`
 
 echo "// Invocation of Kotlin Extractor 2" >> "$INVOCATION_TRAP"
+echo "#compilation = *" >> "$INVOCATION_TRAP"
 
 if [[ -n "$CODEQL_JAVA_HOME" ]]; then
   JAVA="$CODEQL_JAVA_HOME/bin/java"

java/kotlin-extractor2/src/main/kotlin/KotlinExtractor.kt

@@ -69,10 +69,29 @@ fun runExtractor(args : Array<String>) {
         )
 
     // The invocation TRAP file will already have been started
-    // before the plugin is run, so we always use no compression
-    // and we open it in append mode.
+    // as an uncompressed TRAP file before the extractor is run,
+    // so we always use no compression and we open it in append mode.
     FileOutputStream(File(invocationTrapFile), true).bufferedWriter().use { invocationTrapFileBW
         ->
+/*
+OLD: KE1
+        val invocationExtractionProblems = ExtractionProblems()
+*/
+        val invocationLabelManager = TrapLabelManager()
+        val diagnosticCounter = DiagnosticCounter()
+        val loggerBase = LoggerBase(diagnosticCounter)
+        val dtw = DiagnosticTrapWriter(loggerBase, invocationLabelManager, invocationTrapFileBW)
+        // The diganostic TRAP file has already defined #compilation = *
+        val compilation: Label<DbCompilation> = StringLabel("compilation")
+        dtw.writeCompilation_started(compilation)
+/*
+OLD: KE1
+        tw.writeCompilation_info(
+            compilation,
+            "Kotlin Compiler Version",
+            KotlinCompilerVersion.getVersion() ?: "<unknown>"
+        )
+*/
         doAnalysis(kotlinArgs)
     }
 }
@@ -219,19 +238,6 @@ class KotlinExtractorExtension(
     private fun runExtractor(moduleFragment: IrModuleFragment, pluginContext: IrPluginContext) {
         val usesK2 = usesK2(pluginContext)
         [...]
-            val invocationExtractionProblems = ExtractionProblems()
-            val lm = TrapLabelManager()
-            val logCounter = LogCounter()
-            val loggerBase = LoggerBase(logCounter)
-            val tw = DiagnosticTrapWriter(loggerBase, lm, invocationTrapFileBW)
-            // The interceptor has already defined #compilation = *
-            val compilation: Label<DbCompilation> = StringLabel("compilation")
-            tw.writeCompilation_started(compilation)
-            tw.writeCompilation_info(
-                compilation,
-                "Kotlin Compiler Version",
-                KotlinCompilerVersion.getVersion() ?: "<unknown>"
-            )
             val extractor_name =
                 this::class.java.getResource("extractor.name")?.readText() ?: "<unknown>"
             tw.writeCompilation_info(compilation, "Kotlin Extractor Name", extractor_name)

java/kotlin-extractor2/src/main/kotlin/Label.kt

@@ -1,18 +1,22 @@
 package com.github.codeql
 
-/*
-OLD: KE1
 /** This represents a label (`#...`) in a TRAP file. */
 interface Label<T : AnyDbType> {
+/*
+OLD: KE1
     fun <U : AnyDbType> cast(): Label<U> {
         @Suppress("UNCHECKED_CAST") return this as Label<U>
     }
+*/
 }
 
+/*
+OLD: KE1
 /** The label `#i`, e.g. `#123`. Most labels we generate are of this form. */
 class IntLabel<T : AnyDbType>(val i: Int) : Label<T> {
     override fun toString(): String = "#$i"
 }
+*/
 
 /**
  * The label `#name`, e.g. `#compilation`. This is used when labels are shared between different
@@ -21,4 +25,3 @@ class IntLabel<T : AnyDbType>(val i: Int) : Label<T> {
 class StringLabel<T : AnyDbType>(val name: String) : Label<T> {
     override fun toString(): String = "#$name"
 }
-*/

java/kotlin-extractor2/src/main/kotlin/TrapWriter.kt

@@ -4,8 +4,11 @@ package com.github.codeql
 OLD: KE1
 import com.github.codeql.KotlinUsesExtractor.LocallyVisibleFunctionLabels
 import com.semmle.extractor.java.PopulateFile
+*/
 import com.semmle.util.unicode.UTF8Util
 import java.io.BufferedWriter
+/*
+OLD: KE1
 import java.io.File
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
@@ -16,12 +19,15 @@ import org.jetbrains.kotlin.ir.declarations.IrVariable
 import org.jetbrains.kotlin.ir.declarations.path
 import org.jetbrains.kotlin.ir.expressions.IrCall
 import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET
+*/
 
 /**
  * Each `.trap` file has a `TrapLabelManager` while we are writing it. It provides fresh TRAP label
  * names, and maintains a mapping from keys (`@"..."`) to labels.
  */
 class TrapLabelManager {
+/*
+OLD: KE1
     /** The next integer to use as a label name. */
     private var nextInt: Int = 100
 
@@ -52,6 +58,7 @@ class TrapLabelManager {
      * to avoid duplication.
      */
     val fileClassLocationsExtracted = HashSet<IrFile>()
+*/
 }
 
 /**
@@ -59,15 +66,21 @@ class TrapLabelManager {
  * `TrapWriter`s for the same file, as different instances will have different additional state, but
  * they must all share the same `TrapLabelManager` and `BufferedWriter`.
  */
+/*
+OLD: KE1:
 // TODO lm was `protected` before anonymousTypeMapping and locallyVisibleFunctionLabelMapping moved
 // into it. Should we re-protect it and provide accessors?
+Protected again for now in KE2.
+*/
 abstract class TrapWriter(
     protected val loggerBase: LoggerBase,
-    val lm: TrapLabelManager,
+    protected val lm: TrapLabelManager,
     private val bw: BufferedWriter
 ) {
     abstract fun getDiagnosticTrapWriter(): DiagnosticTrapWriter
 
+/*
+OLD: KE1
     /**
      * Returns the label that is defined to be the given key, if such a label exists, and `null`
      * otherwise. Most users will want to use `getLabelFor` instead, which allows non-existent
@@ -185,15 +198,19 @@ abstract class TrapWriter(
     fun getWholeFileLocation(fileId: Label<DbFile>): Label<DbLocation> {
         return getLocation(fileId, 0, 0, 0, 0)
     }
+*/
 
     /**
-     * Write a raw string into the TRAP file. Users should call one of the wrapper functions
-     * instead.
+     * Write a raw string into the TRAP file.
+     * The only external caller of this should be the generated
+     * KotlinExtractorDbScheme.kt.
      */
     fun writeTrap(trap: String) {
         bw.write(trap)
     }
 
+/*
+OLD: KE1
     /** Write a comment into the TRAP file. */
     fun writeComment(comment: String) {
         writeTrap("// ${comment.replace("\n", "\n//    ")}\n")
@@ -203,10 +220,13 @@ abstract class TrapWriter(
     fun flush() {
         bw.flush()
     }
+*/
 
     /**
      * Escape a string so that it can be used in a TRAP string literal, i.e. with `"` escaped as
      * `""`.
+     * The only external caller of this should be the generated
+     * KotlinExtractorDbScheme.kt.
      */
     fun escapeTrapString(str: String) = str.replace("\"", "\"\"")
 
@@ -216,6 +236,8 @@ abstract class TrapWriter(
     /**
      * Truncate a string, if necessary, so that it can be used as a TRAP string literal. TRAP string
      * literals are limited to 1 megabyte.
+     * The only external caller of this should be the generated
+     * KotlinExtractorDbScheme.kt.
      */
     fun truncateString(str: String): String {
         val len = str.length
@@ -232,6 +254,8 @@ abstract class TrapWriter(
         }
     }
 
+/*
+OLD: KE1
     /**
      * Gets a FileTrapWriter like this one (using the same label manager, writer etc), but using the
      * given `filePath` for locations.
@@ -259,8 +283,11 @@ abstract class TrapWriter(
             file,
             populateFileTables
         )
+*/
 }
 
+/*
+OLD: KE1
 /** A `PlainTrapWriter` has no additional context of its own. */
 class PlainTrapWriter(
     loggerBase: LoggerBase,
@@ -272,6 +299,7 @@ class PlainTrapWriter(
         return dtw
     }
 }
+*/
 
 /**
  * A `DiagnosticTrapWriter` is a TrapWriter that diagnostics can be written to; i.e. it has
@@ -285,6 +313,8 @@ class DiagnosticTrapWriter(loggerBase: LoggerBase, lm: TrapLabelManager, bw: Buf
     }
 }
 
+/*
+OLD: KE1
 /**
  * A `FileTrapWriter` is used when we know which file we are extracting entities from, so we can at
  * least give the right file as a location.

java/kotlin-extractor2/src/main/kotlin/utils/Logger.kt

@@ -100,9 +100,19 @@ data class ExtractorContext(
     val name: String,
     val loc: String
 )
+*/
 
-open class LoggerBase(val logCounter: LogCounter) {
+/**
+ * LoggerBase actually writes log messages to a log file, and to
+ * the DiagnosticTrapWriter that it is passed.
+ * It is only usde directly from the DiagnosticTrapWriter. Everything
+ * else will use a Logger that wraps it (and the DiagnosticTrapWriter).
+ */
+open class LoggerBase(val diagnosticCounter: DiagnosticCounter) {
+/*
+OLD: KE1
     val extractorContextStack = Stack<ExtractorContext>()
+*/
 
     private val verbosity: Int
 
@@ -110,6 +120,8 @@ open class LoggerBase(val logCounter: LogCounter) {
         verbosity = System.getenv("CODEQL_EXTRACTOR_KOTLIN_VERBOSITY")?.toIntOrNull() ?: 3
     }
 
+/*
+OLD: KE1
     private val logStream: Writer
 
     init {
@@ -256,13 +268,19 @@ open class LoggerBase(val logCounter: LogCounter) {
             logStream.write(logMessage.toJsonLine())
         }
     }
+*/
 
     fun warn(dtw: DiagnosticTrapWriter, msg: String, extraInfo: String?) {
         if (verbosity >= 2) {
+/*
+OLD: KE1
             diagnostic(dtw, Severity.Warn, msg, extraInfo)
+*/
         }
     }
 
+/*
+OLD: KE1
     fun error(dtw: DiagnosticTrapWriter, msg: String, extraInfo: String?) {
         if (verbosity >= 1) {
             diagnostic(dtw, Severity.Error, msg, extraInfo)
@@ -290,8 +308,11 @@ open class LoggerBase(val logCounter: LogCounter) {
     fun close() {
         logStream.close()
     }
+*/
 }
 
+/*
+OLD: KE1
 open class Logger(val loggerBase: LoggerBase, open val dtw: DiagnosticTrapWriter) {
     fun flush() {
         dtw.flush()