Split main extractor file

java/kotlin-extractor/src/main/kotlin/ExternalClassExtractor.kt

@@ -0,0 +1,48 @@
+package com.github.codeql
+
+import com.semmle.extractor.java.OdasaOutput
+import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
+import org.jetbrains.kotlin.ir.declarations.IrClass
+import java.io.File
+import java.util.ArrayList
+import java.util.HashSet
+import java.util.zip.GZIPOutputStream
+
+class ExternalClassExtractor(val logger: FileLogger, val sourceFilePath: String, val pluginContext: IrPluginContext) {
+
+    val externalClassesDone = HashSet<IrClass>()
+    val externalClassWorkList = ArrayList<IrClass>()
+
+    fun extractLater(c: IrClass): Boolean {
+        val ret = externalClassesDone.add(c)
+        if(ret) externalClassWorkList.add(c)
+        return ret
+    }
+
+    fun extractExternalClasses() {
+        val output = OdasaOutput(false, logger)
+        output.setCurrentSourceFile(File(sourceFilePath))
+        do {
+            val nextBatch = ArrayList(externalClassWorkList)
+            externalClassWorkList.clear()
+            nextBatch.forEach { irClass ->
+                output.getTrapLockerForClassFile(irClass).useAC { locker ->
+                    locker.getTrapFileManager().useAC { manager ->
+                        if(manager == null) {
+                            logger.info("Skipping extracting class ${irClass.name}")
+                        } else {
+                            GZIPOutputStream(manager.getFile().outputStream()).bufferedWriter().use { trapFileBW ->
+                                val tw =
+                                    ClassFileTrapWriter(TrapLabelManager(), trapFileBW, getIrClassBinaryPath(irClass))
+                                val fileExtractor = KotlinFileExtractor(logger, tw, manager, this, pluginContext)
+                                fileExtractor.extractClassSource(irClass)
+                            }
+                        }
+                    }
+                }
+            }
+        } while (externalClassWorkList.isNotEmpty())
+        output.writeTrapSet()
+    }
+
+}

java/kotlin-extractor/src/main/kotlin/KotlinSourceFileExtractor.kt

@@ -0,0 +1,68 @@
+package com.github.codeql
+
+import com.github.codeql.comments.CommentExtractor
+import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
+import org.jetbrains.kotlin.ir.declarations.IrClass
+import org.jetbrains.kotlin.ir.declarations.IrFile
+import org.jetbrains.kotlin.ir.expressions.IrConst
+import org.jetbrains.kotlin.ir.expressions.IrConstructorCall
+import org.jetbrains.kotlin.ir.types.IrSimpleType
+import org.jetbrains.kotlin.ir.util.packageFqName
+
+class KotlinSourceFileExtractor(
+    logger: FileLogger,
+    tw: FileTrapWriter,
+    val file: IrFile,
+    externalClassExtractor: ExternalClassExtractor,
+    pluginContext: IrPluginContext
+) :
+  KotlinFileExtractor(logger, tw, null, externalClassExtractor, pluginContext) {
+
+    val fileClass by lazy {
+        extractFileClass(file)
+    }
+
+    fun extractFileContents(id: Label<DbFile>) {
+        val locId = tw.getWholeFileLocation()
+        val pkg = file.fqName.asString()
+        val pkgId = extractPackage(pkg)
+        tw.writeHasLocation(id, locId)
+        tw.writeCupackage(id, pkgId)
+        file.declarations.map { extractDeclaration(it, fileClass) }
+        CommentExtractor(this).extract()
+    }
+
+  @OptIn(kotlin.ExperimentalStdlibApi::class) // Annotation required by kotlin versions < 1.5
+  fun extractFileClass(f: IrFile): Label<out DbClass> {
+      val fileName = f.fileEntry.name
+      val pkg = f.fqName.asString()
+      val defaultName = fileName.replaceFirst(Regex(""".*[/\\]"""), "").replaceFirst(Regex("""\.kt$"""), "").replaceFirstChar({ it.uppercase() }) + "Kt"
+      var jvmName = defaultName
+      for(a: IrConstructorCall in f.annotations) {
+          val t = a.type
+          if(t is IrSimpleType && a.valueArgumentsCount == 1) {
+              val owner = t.classifier.owner
+              val v = a.getValueArgument(0)
+              if(owner is IrClass) {
+                  val aPkg = owner.packageFqName?.asString()
+                  val name = owner.name.asString()
+                  if(aPkg == "kotlin.jvm" && name == "JvmName" && v is IrConst<*>) {
+                      val value = v.value
+                      if(value is String) {
+                          jvmName = value
+                      }
+                  }
+              }
+          }
+      }
+      val qualClassName = if (pkg.isEmpty()) jvmName else "$pkg.$jvmName"
+      val label = "@\"class;$qualClassName\""
+      val id: Label<DbClass> = tw.getLabelFor(label)
+      val locId = tw.getWholeFileLocation()
+      val pkgId = extractPackage(pkg)
+      tw.writeClasses(id, jvmName, pkgId, id)
+      tw.writeHasLocation(id, locId)
+      return id
+  }
+
+}

java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt

@@ -0,0 +1,681 @@
+package com.github.codeql
+
+import com.semmle.extractor.java.OdasaOutput
+import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
+import org.jetbrains.kotlin.builtins.jvm.JavaToKotlinClassMap
+import org.jetbrains.kotlin.descriptors.Modality
+import org.jetbrains.kotlin.ir.declarations.*
+import org.jetbrains.kotlin.ir.symbols.IrClassSymbol
+import org.jetbrains.kotlin.ir.symbols.IrClassifierSymbol
+import org.jetbrains.kotlin.ir.types.*
+import org.jetbrains.kotlin.ir.types.impl.IrSimpleTypeImpl
+import org.jetbrains.kotlin.ir.types.impl.makeTypeProjection
+import org.jetbrains.kotlin.ir.util.*
+import org.jetbrains.kotlin.types.Variance
+
+open class KotlinUsesExtractor(
+    open val logger: Logger,
+    open val tw: TrapWriter,
+    val dependencyCollector: OdasaOutput.TrapFileManager?,
+    val externalClassExtractor: ExternalClassExtractor,
+    val pluginContext: IrPluginContext
+) {
+    fun usePackage(pkg: String): Label<out DbPackage> {
+        return extractPackage(pkg)
+    }
+
+    fun extractPackage(pkg: String): Label<out DbPackage> {
+        val pkgLabel = "@\"package;$pkg\""
+        val id: Label<DbPackage> = tw.getLabelFor(pkgLabel, {
+            tw.writePackages(it, pkg)
+        })
+        return id
+    }
+
+    data class UseClassInstanceResult(val typeResult: TypeResult<DbClassorinterface>, val javaClass: IrClass)
+    /**
+     * A triple of a type's database label, its signature for use in callable signatures, and its short name for use
+     * in all tables that provide a user-facing type name.
+     *
+     * `signature` is a Java primitive name (e.g. "int"), a fully-qualified class name ("package.OuterClass.InnerClass"),
+     * or an array ("componentSignature[]")
+     * Type variables have the signature of their upper bound.
+     * Type arguments and anonymous types do not have a signature.
+     *
+     * `shortName` is a Java primitive name (e.g. "int"), a class short name with Java-style type arguments ("InnerClass<E>" or
+     * "OuterClass<ConcreteArgument>" or "OtherClass<? extends Bound>") or an array ("componentShortName[]").
+     */
+    data class TypeResult<out LabelType>(val id: Label<out LabelType>, val signature: String?, val shortName: String)
+    data class TypeResults(val javaResult: TypeResult<DbType>, val kotlinResult: TypeResult<DbKt_type>)
+
+    fun useType(t: IrType, canReturnPrimitiveTypes: Boolean = true) =
+        when(t) {
+            is IrSimpleType -> useSimpleType(t, canReturnPrimitiveTypes)
+            else -> {
+                logger.warn(Severity.ErrorSevere, "Unrecognised IrType: " + t.javaClass)
+                TypeResults(TypeResult(fakeLabel(), "unknown", "unknown"), TypeResult(fakeLabel(), "unknown", "unknown"))
+            }
+        }
+
+    fun getJavaEquivalentClass(c: IrClass) =
+        c.fqNameWhenAvailable?.toUnsafe()
+            ?.let { JavaToKotlinClassMap.mapKotlinToJava(it) }
+            ?.let { pluginContext.referenceClass(it.asSingleFqName()) }
+            ?.owner
+
+    /**
+     * Gets a KotlinFileExtractor based on this one, except it attributes locations to the file that declares the given class.
+     */
+    fun withSourceFileOfClass(cls: IrClass, populateFileTables: Boolean): KotlinFileExtractor {
+        val clsFile = cls.fileOrNull
+
+        val newTrapWriter =
+            if (isExternalDeclaration(cls) || clsFile == null)
+                tw.withTargetFile(getIrClassBinaryPath(cls), null, populateFileTables)
+            else
+                tw.withTargetFile(clsFile.path, clsFile.fileEntry)
+
+        val newLogger = FileLogger(logger.logCounter, newTrapWriter)
+
+        return KotlinFileExtractor(newLogger, newTrapWriter, dependencyCollector, externalClassExtractor, pluginContext)
+    }
+
+    fun useClassInstance(c: IrClass, typeArgs: List<IrTypeArgument>): UseClassInstanceResult {
+        if (c.isAnonymousObject) {
+            logger.warn(Severity.ErrorSevere, "Unexpected access to anonymous class instance")
+        }
+
+        // TODO: only substitute in class and function signatures
+        //       because within function bodies we can get things like Unit.INSTANCE
+        //       and List.asIterable (an extension, i.e. static, method)
+        // Map Kotlin class to its equivalent Java class:
+        val substituteClass = getJavaEquivalentClass(c)
+
+        val extractClass = substituteClass ?: c
+
+        val classId = getClassLabel(extractClass, typeArgs)
+        val classLabel : Label<out DbClassorinterface> = tw.getLabelFor(classId.classLabel, {
+            // If this is a generic type instantiation then it has no
+            // source entity, so we need to extract it here
+            if (typeArgs.isNotEmpty()) {
+                this.withSourceFileOfClass(extractClass, false).extractClassInstance(extractClass, typeArgs)
+            }
+
+            // Extract both the Kotlin and equivalent Java classes, so that we have database entries
+            // for both even if all internal references to the Kotlin type are substituted.
+            extractClassLaterIfExternal(c)
+            substituteClass?.let { extractClassLaterIfExternal(it) }
+        })
+
+        return UseClassInstanceResult(TypeResult(classLabel, extractClass.fqNameWhenAvailable?.asString(), classId.shortName), extractClass)
+    }
+
+    fun isExternalDeclaration(d: IrDeclaration): Boolean {
+        return d.origin == IrDeclarationOrigin.IR_EXTERNAL_DECLARATION_STUB ||
+               d.origin == IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB
+    }
+
+    fun isArray(t: IrSimpleType) = t.isBoxedArray || t.isPrimitiveArray()
+
+    fun extractClassLaterIfExternal(c: IrClass) {
+        if (isExternalDeclaration(c)) {
+            extractExternalClassLater(c)
+        }
+    }
+
+    fun extractExternalEnclosingClassLater(d: IrDeclaration) {
+        when (val parent = d.parent) {
+            is IrClass -> extractExternalClassLater(parent)
+            is IrFunction -> extractExternalEnclosingClassLater(parent)
+            is IrFile -> logger.warn(Severity.ErrorSevere, "extractExternalEnclosingClassLater but no enclosing class.")
+            else -> logger.warn(Severity.ErrorSevere, "Unrecognised extractExternalEnclosingClassLater: " + d.javaClass)
+        }
+    }
+
+    fun extractExternalClassLater(c: IrClass) {
+        dependencyCollector?.addDependency(c)
+        externalClassExtractor.extractLater(c)
+    }
+
+    fun addClassLabel(c: IrClass, typeArgs: List<IrTypeArgument>): TypeResult<DbClassorinterface> {
+        val classLabelResult = getClassLabel(c, typeArgs)
+        return TypeResult(
+            tw.getLabelFor(classLabelResult.classLabel),
+            c.fqNameWhenAvailable?.asString(),
+            classLabelResult.shortName)
+    }
+
+    open fun useAnonymousClass(c: IrClass): TypeResults {
+        throw Exception("Anonymous classes can only be accessed through source file extraction")
+    }
+
+    fun useSimpleTypeClass(c: IrClass, args: List<IrTypeArgument>, hasQuestionMark: Boolean): TypeResults {
+        if (c.isAnonymousObject) {
+            if (args.isNotEmpty()) {
+                logger.warn(Severity.ErrorHigh, "Anonymous class with unexpected type arguments")
+            }
+            if (hasQuestionMark) {
+                logger.warn(Severity.ErrorHigh, "Unexpected nullable anonymous class")
+            }
+
+            return useAnonymousClass(c)
+        }
+
+        val classInstanceResult = useClassInstance(c, args)
+        val javaClassId = classInstanceResult.typeResult.id
+        val kotlinQualClassName = getUnquotedClassLabel(c, args).classLabel
+        // TODO: args ought to be substituted, so e.g. MutableList<MutableList<String>> gets the Java type List<List<String>>
+        val javaResult = classInstanceResult.typeResult
+        val kotlinResult = if (hasQuestionMark) {
+                val kotlinSignature = "$kotlinQualClassName?" // TODO: Is this right?
+                val kotlinLabel = "@\"kt_type;nullable;$kotlinQualClassName\""
+                val kotlinId: Label<DbKt_nullable_type> = tw.getLabelFor(kotlinLabel, {
+                    tw.writeKt_nullable_types(it, javaClassId)
+                })
+                TypeResult(kotlinId, kotlinSignature, "TODO")
+            } else {
+                val kotlinSignature = kotlinQualClassName // TODO: Is this right?
+                val kotlinLabel = "@\"kt_type;notnull;$kotlinQualClassName\""
+                val kotlinId: Label<DbKt_notnull_type> = tw.getLabelFor(kotlinLabel, {
+                    tw.writeKt_notnull_types(it, javaClassId)
+                })
+                TypeResult(kotlinId, kotlinSignature, "TODO")
+            }
+        return TypeResults(javaResult, kotlinResult)
+    }
+
+    // Given either a primitive array or a boxed array, returns primitive arrays unchanged,
+    // but returns boxed arrays with a nullable, invariant component type, with any nested arrays
+    // similarly transformed. For example, Array<out Array<in E>> would become Array<Array<E?>?>
+    // Array<*> will become Array<Any?>.
+    fun getInvariantNullableArrayType(arrayType: IrSimpleType): IrSimpleType =
+        if (arrayType.isPrimitiveArray())
+            arrayType
+        else {
+            val componentType = arrayType.getArrayElementType(pluginContext.irBuiltIns)
+            val componentTypeBroadened = when (componentType) {
+                is IrSimpleType ->
+                    if (isArray(componentType)) getInvariantNullableArrayType(componentType) else componentType
+                else -> componentType
+            }
+            val unchanged =
+                componentType == componentTypeBroadened &&
+                        (arrayType.arguments[0] as? IrTypeProjection)?.variance == Variance.INVARIANT &&
+                        componentType.isNullable()
+            if (unchanged)
+                arrayType
+            else
+                IrSimpleTypeImpl(
+                    arrayType.classifier,
+                    true,
+                    listOf(makeTypeProjection(componentTypeBroadened, Variance.INVARIANT)),
+                    listOf()
+                )
+        }
+
+    fun useArrayType(arrayType: IrSimpleType, componentType: IrType, elementType: IrType, dimensions: Int, isPrimitiveArray: Boolean): TypeResults {
+
+        // Ensure we extract Array<Int> as Integer[], not int[], for example:
+        fun nullableIfNotPrimitive(type: IrType) = if (type.isPrimitiveType() && !isPrimitiveArray) type.makeNullable() else type
+
+        // TODO: Figure out what signatures should be returned
+
+        val componentTypeResults = useType(nullableIfNotPrimitive(componentType))
+        val elementTypeLabel = useType(nullableIfNotPrimitive(elementType)).javaResult.id
+
+        val javaShortName = componentTypeResults.javaResult.shortName + "[]"
+
+        val id = tw.getLabelFor<DbArray>("@\"array;$dimensions;{${elementTypeLabel}}\"") {
+            tw.writeArrays(
+                it,
+                javaShortName,
+                elementTypeLabel,
+                dimensions,
+                componentTypeResults.javaResult.id)
+
+            extractClassSupertypes(arrayType.classifier.owner as IrClass, it)
+
+            // array.length
+            val length = tw.getLabelFor<DbField>("@\"field;{$it};length\"")
+            val intTypeIds = useType(pluginContext.irBuiltIns.intType)
+            tw.writeFields(length, "length", intTypeIds.javaResult.id, intTypeIds.kotlinResult.id, it, length)
+            addModifiers(length, "public", "final")
+
+            // Note we will only emit one `clone()` method per Java array type, so we choose `Array<C?>` as its Kotlin
+            // return type, where C is the component type with any nested arrays themselves invariant and nullable.
+            val kotlinCloneReturnType = getInvariantNullableArrayType(arrayType).makeNullable()
+            val kotlinCloneReturnTypeLabel = useType(kotlinCloneReturnType).kotlinResult.id
+
+            val clone = tw.getLabelFor<DbMethod>("@\"callable;{$it}.clone(){$it}\"")
+            tw.writeMethods(clone, "clone", "clone()", it, kotlinCloneReturnTypeLabel, it, clone)
+            addModifiers(clone, "public")
+        }
+
+        val javaResult = TypeResult(
+            id,
+            componentTypeResults.javaResult.signature + "[]",
+            javaShortName)
+
+        val arrayClassResult = useSimpleTypeClass(arrayType.classifier.owner as IrClass, arrayType.arguments, arrayType.hasQuestionMark)
+        return TypeResults(javaResult, arrayClassResult.kotlinResult)
+    }
+
+    fun useSimpleType(s: IrSimpleType, canReturnPrimitiveTypes: Boolean): TypeResults {
+        if (s.abbreviation != null) {
+            // TODO: Extract this information
+            logger.warn(Severity.ErrorSevere, "Type alias ignored for " + s.render())
+        }
+        // We use this when we don't actually have an IrClass for a class
+        // we want to refer to
+        fun makeClass(pkgName: String, className: String): Label<DbClass> {
+            val pkgId = extractPackage(pkgName)
+            val label = "@\"class;$pkgName.$className\""
+            val classId: Label<DbClass> = tw.getLabelFor(label, {
+                tw.writeClasses(it, className, pkgId, it)
+            })
+            return classId
+        }
+        fun primitiveType(kotlinClass: IrClass, primitiveName: String?,
+                          javaPackageName: String, javaClassName: String,
+                          kotlinPackageName: String, kotlinClassName: String): TypeResults {
+            val javaResult = if (canReturnPrimitiveTypes && !s.hasQuestionMark && primitiveName != null) {
+                    val label: Label<DbPrimitive> = tw.getLabelFor("@\"type;$primitiveName\"", {
+                        tw.writePrimitives(it, primitiveName)
+                    })
+                    TypeResult(label, primitiveName, primitiveName)
+                } else {
+                    val label = makeClass(javaPackageName, javaClassName)
+                    val signature = "$javaPackageName.$javaClassName"
+                    TypeResult(label, signature, javaClassName)
+                }
+            val kotlinClassId = useClassInstance(kotlinClass, listOf()).typeResult.id
+            val kotlinResult = if (s.hasQuestionMark) {
+                    val kotlinSignature = "$kotlinPackageName.$kotlinClassName?" // TODO: Is this right?
+                    val kotlinLabel = "@\"kt_type;nullable;$kotlinPackageName.$kotlinClassName\""
+                    val kotlinId: Label<DbKt_nullable_type> = tw.getLabelFor(kotlinLabel, {
+                        tw.writeKt_nullable_types(it, kotlinClassId)
+                    })
+                    TypeResult(kotlinId, kotlinSignature, "TODO")
+                } else {
+                    val kotlinSignature = "$kotlinPackageName.$kotlinClassName" // TODO: Is this right?
+                    val kotlinLabel = "@\"kt_type;notnull;$kotlinPackageName.$kotlinClassName\""
+                    val kotlinId: Label<DbKt_notnull_type> = tw.getLabelFor(kotlinLabel, {
+                        tw.writeKt_notnull_types(it, kotlinClassId)
+                    })
+                    TypeResult(kotlinId, kotlinSignature, "TODO")
+                }
+            return TypeResults(javaResult, kotlinResult)
+        }
+
+        val primitiveInfo = primitiveTypeMapping[s.classifier.signature]
+
+        when {
+/*
+XXX delete?
+            // temporary fix for type parameters types that would otherwise be primitive types
+            !canReturnPrimitiveTypes && (s.isPrimitiveType() || s.isUnsignedType() || s.isString()) -> {
+                val classifier: IrClassifierSymbol = s.classifier
+                val cls: IrClass = classifier.owner as IrClass
+
+                return useClassInstance(cls, s.arguments)
+            }
+
+*/
+            primitiveInfo != null -> return primitiveType(
+                s.classifier.owner as IrClass,
+                primitiveInfo.primitiveName, primitiveInfo.javaPackageName,
+                primitiveInfo.javaClassName, primitiveInfo.kotlinPackageName, primitiveInfo.kotlinClassName
+            )
+/*
+TODO: Test case: nullable and has-question-mark type variables:
+class X {
+    fun <T : Int> f1(t: T?) {
+        f1(null)
+    }
+
+    fun <T : Int?> f2(t: T) {
+        f2(null)
+    }
+}
+
+TODO: Test case: This breaks kotlinc codegen currently, but up to IR is OK, so we can still have it in a qltest
+class X {
+    fun <T : Int> f1(t: T?) {
+        f1(null)
+    }
+}
+*/
+
+            (s.isBoxedArray && s.arguments.isNotEmpty()) || s.isPrimitiveArray() -> {
+                var dimensions = 1
+                var isPrimitiveArray = s.isPrimitiveArray()
+                val componentType = s.getArrayElementType(pluginContext.irBuiltIns)
+                var elementType = componentType
+                while (elementType.isBoxedArray || elementType.isPrimitiveArray()) {
+                    dimensions++
+                    if(elementType.isPrimitiveArray())
+                        isPrimitiveArray = true
+                    elementType = elementType.getArrayElementType(pluginContext.irBuiltIns)
+                }
+
+                return useArrayType(
+                    s,
+                    componentType,
+                    elementType,
+                    dimensions,
+                    isPrimitiveArray
+                )
+            }
+
+            s.classifier.owner is IrClass -> {
+                val classifier: IrClassifierSymbol = s.classifier
+                val cls: IrClass = classifier.owner as IrClass
+
+                return useSimpleTypeClass(cls, s.arguments, s.hasQuestionMark)
+            }
+            s.classifier.owner is IrTypeParameter -> {
+                val javaResult = useTypeParameter(s.classifier.owner as IrTypeParameter)
+                val aClassId = makeClass("kotlin", "TypeParam") // TODO: Wrong
+                val kotlinResult = if (s.hasQuestionMark) {
+                        val kotlinSignature = "${javaResult.signature}?" // TODO: Wrong
+                        val kotlinLabel = "@\"kt_type;nullable;type_param\"" // TODO: Wrong
+                        val kotlinId: Label<DbKt_nullable_type> = tw.getLabelFor(kotlinLabel, {
+                            tw.writeKt_nullable_types(it, aClassId)
+                        })
+                        TypeResult(kotlinId, kotlinSignature, "TODO")
+                    } else {
+                        val kotlinSignature = javaResult.signature // TODO: Wrong
+                        val kotlinLabel = "@\"kt_type;notnull;type_param\"" // TODO: Wrong
+                        val kotlinId: Label<DbKt_notnull_type> = tw.getLabelFor(kotlinLabel, {
+                            tw.writeKt_notnull_types(it, aClassId)
+                        })
+                        TypeResult(kotlinId, kotlinSignature, "TODO")
+                    }
+                return TypeResults(javaResult, kotlinResult)
+            }
+            else -> {
+                logger.warn(Severity.ErrorSevere, "Unrecognised IrSimpleType: " + s.javaClass + ": " + s.render())
+                return TypeResults(TypeResult(fakeLabel(), "unknown", "unknown"), TypeResult(fakeLabel(), "unknown", "unknown"))
+            }
+        }
+    }
+
+    fun useDeclarationParent(dp: IrDeclarationParent): Label<out DbElement> =
+        when(dp) {
+            is IrFile -> usePackage(dp.fqName.asString())
+            is IrClass -> useClassSource(dp)
+            is IrFunction -> useFunction(dp)
+            else -> {
+                logger.warn(Severity.ErrorSevere, "Unrecognised IrDeclarationParent: " + dp.javaClass)
+                fakeLabel()
+            }
+        }
+
+    fun getFunctionLabel(f: IrFunction) : String {
+        return getFunctionLabel(f.parent, f.name.asString(), f.valueParameters, f.returnType)
+    }
+
+    fun getFunctionLabel(
+        parent: IrDeclarationParent,
+        name: String,
+        parameters: List<IrValueParameter>,
+        returnType: IrType
+    ): String {
+        val paramTypeIds = parameters.joinToString { "{${useType(erase(it.type)).javaResult.id}}" }
+        val returnTypeId = useType(erase(returnType)).javaResult.id
+        val parentId = useDeclarationParent(parent)
+        return "@\"callable;{$parentId}.$name($paramTypeIds){$returnTypeId}\""
+    }
+
+    fun <T: DbCallable> useFunction(f: IrFunction): Label<out T> {
+        val label = getFunctionLabel(f)
+        val id: Label<T> = tw.getLabelFor(label)
+        if(isExternalDeclaration(f)) {
+            extractExternalEnclosingClassLater(f)
+        }
+        return id
+    }
+
+    fun getTypeArgumentLabel(
+        arg: IrTypeArgument
+    ): TypeResult<DbReftype> {
+
+        fun extractBoundedWildcard(wildcardKind: Int, wildcardLabelStr: String, wildcardShortName: String, boundLabel: Label<out DbReftype>): Label<DbWildcard> =
+            tw.getLabelFor(wildcardLabelStr) { wildcardLabel ->
+                tw.writeWildcards(wildcardLabel, wildcardShortName, wildcardKind)
+                tw.writeHasLocation(wildcardLabel, tw.unknownLocation)
+                tw.getLabelFor<DbTypebound>("@\"bound;0;{$wildcardLabel}\"") {
+                    tw.writeTypeBounds(it, boundLabel, 0, wildcardLabel)
+                }
+            }
+
+        // Note this function doesn't return a signature because type arguments are never incorporated into function signatures.
+        return when (arg) {
+            is IrStarProjection -> {
+                @Suppress("UNCHECKED_CAST")
+                val anyTypeLabel = useType(pluginContext.irBuiltIns.anyType).javaResult.id as Label<out DbReftype>
+                TypeResult(extractBoundedWildcard(1, "@\"wildcard;\"", "?", anyTypeLabel), null, "?")
+            }
+            is IrTypeProjection -> {
+                val boundResults = useType(arg.type, false)
+                @Suppress("UNCHECKED_CAST")
+                val boundLabel = boundResults.javaResult.id as Label<out DbReftype>
+
+                return if(arg.variance == Variance.INVARIANT)
+                    @Suppress("UNCHECKED_CAST")
+                    boundResults.javaResult as TypeResult<DbReftype>
+                else {
+                    val keyPrefix = if (arg.variance == Variance.IN_VARIANCE) "super" else "extends"
+                    val wildcardKind = if (arg.variance == Variance.IN_VARIANCE) 2 else 1
+                    val wildcardShortName = "? $keyPrefix ${boundResults.javaResult.shortName}"
+                    TypeResult(
+                        extractBoundedWildcard(wildcardKind, "@\"wildcard;$keyPrefix{$boundLabel}\"", wildcardShortName, boundLabel),
+                        null,
+                        wildcardShortName)
+                }
+            }
+            else -> {
+                logger.warn(Severity.ErrorSevere, "Unexpected type argument.")
+                return TypeResult(fakeLabel(), "unknown", "unknown")
+            }
+        }
+    }
+
+    data class ClassLabelResults(
+        val classLabel: String, val shortName: String
+    )
+
+    /*
+    This returns the `X` in c's label `@"class;X"`.
+    */
+    private fun getUnquotedClassLabel(c: IrClass, typeArgs: List<IrTypeArgument>): ClassLabelResults {
+        val pkg = c.packageFqName?.asString() ?: ""
+        val cls = c.name.asString()
+        val parent = c.parent
+        val label = if (parent is IrClass) {
+            // todo: fix this. Ugly string concat to handle nested class IDs.
+            // todo: Can the containing class have type arguments?
+            "${getUnquotedClassLabel(parent, listOf())}\$$cls"
+        } else {
+            if (pkg.isEmpty()) cls else "$pkg.$cls"
+        }
+
+        val typeArgLabels = typeArgs.map { getTypeArgumentLabel(it) }
+        val typeArgsShortName =
+            if(typeArgs.isEmpty())
+                ""
+            else
+                typeArgLabels.joinToString(prefix = "<", postfix = ">", separator = ",") { it.shortName }
+
+        return ClassLabelResults(
+            label + typeArgLabels.joinToString(separator = "") { ";{${it.id}}" },
+            cls + typeArgsShortName
+        )
+    }
+
+
+    fun getClassLabel(c: IrClass, typeArgs: List<IrTypeArgument>): ClassLabelResults {
+        if (c.isAnonymousObject) {
+            logger.warn(Severity.ErrorSevere, "Label generation should not be requested for an anonymous class")
+        }
+
+        val unquotedLabel = getUnquotedClassLabel(c, typeArgs)
+        return ClassLabelResults(
+            "@\"class;${unquotedLabel.classLabel}\"",
+            unquotedLabel.shortName)
+    }
+
+    fun useClassSource(c: IrClass): Label<out DbClassorinterface> {
+        if (c.isAnonymousObject) {
+            @Suppress("UNCHECKED_CAST")
+            return useAnonymousClass(c).javaResult.id as Label<DbClass>
+        }
+
+        // For source classes, the label doesn't include and type arguments
+        val classId = getClassLabel(c, listOf())
+        return tw.getLabelFor(classId.classLabel)
+    }
+
+    fun getTypeParameterLabel(param: IrTypeParameter): String {
+        val parentLabel = useDeclarationParent(param.parent)
+        return "@\"typevar;{$parentLabel};${param.name}\""
+    }
+
+    fun useTypeParameter(param: IrTypeParameter) =
+        TypeResult(
+            tw.getLabelFor<DbTypevariable>(getTypeParameterLabel(param)) {
+                // Any type parameter that is in scope should have been extracted already
+                // in extractClassSource or extractFunction
+                logger.warn(Severity.ErrorSevere, "Missing type parameter label")
+            },
+            useType(eraseTypeParameter(param)).javaResult.signature,
+            param.name.asString()
+        )
+
+    fun extractModifier(m: String): Label<DbModifier> {
+        val modifierLabel = "@\"modifier;$m\""
+        val id: Label<DbModifier> = tw.getLabelFor(modifierLabel, {
+            tw.writeModifiers(it, m)
+        })
+        return id
+    }
+
+    fun addModifiers(modifiable: Label<out DbModifiable>, vararg modifiers: String) =
+        modifiers.forEach { tw.writeHasModifier(modifiable, extractModifier(it)) }
+
+    fun extractClassModifiers(c: IrClass, id: Label<out DbClassorinterface>) {
+        if (c.modality == Modality.ABSTRACT) {
+            addModifiers(id, "abstract")
+        }
+    }
+
+    fun extractClassSupertypes(c: IrClass, id: Label<out DbReftype>) {
+        for(t in c.superTypes) {
+            when(t) {
+                is IrSimpleType -> {
+                    when (t.classifier.owner) {
+                        is IrClass -> {
+                            val classifier: IrClassifierSymbol = t.classifier
+                            val tcls: IrClass = classifier.owner as IrClass
+                            val l = useClassInstance(tcls, t.arguments).typeResult.id
+                            tw.writeExtendsReftype(id, l)
+                        }
+                        else -> {
+                            logger.warn(Severity.ErrorSevere, "Unexpected simple type supertype: " + t.javaClass + ": " + t.render())
+                        }
+                    }
+                } else -> {
+                    logger.warn(Severity.ErrorSevere, "Unexpected supertype: " + t.javaClass + ": " + t.render())
+                }
+            }
+        }
+    }
+
+    fun useValueDeclaration(d: IrValueDeclaration): Label<out DbVariable> =
+        when(d) {
+            is IrValueParameter -> useValueParameter(d)
+            is IrVariable -> useVariable(d)
+            else -> {
+                logger.warn(Severity.ErrorSevere, "Unrecognised IrValueDeclaration: " + d.javaClass)
+                fakeLabel()
+            }
+        }
+
+    fun erase (t: IrType): IrType {
+        if (t is IrSimpleType) {
+            val classifier = t.classifier
+            val owner = classifier.owner
+            if(owner is IrTypeParameter) {
+                return eraseTypeParameter(owner)
+            }
+
+            // todo: fix this:
+            if (t.makeNotNull().isArray()) {
+                val elementType = t.getArrayElementType(pluginContext.irBuiltIns)
+                val erasedElementType = erase(elementType)
+                return withQuestionMark((classifier as IrClassSymbol).typeWith(erasedElementType), t.hasQuestionMark)
+            }
+
+            if (owner is IrClass) {
+                return withQuestionMark((classifier as IrClassSymbol).typeWith(), t.hasQuestionMark)
+            }
+        }
+        return t
+    }
+
+    fun eraseTypeParameter(t: IrTypeParameter) =
+        erase(t.superTypes[0])
+
+    fun getValueParameterLabel(vp: IrValueParameter): String {
+        @Suppress("UNCHECKED_CAST")
+        val parentId: Label<out DbMethod> = useDeclarationParent(vp.parent) as Label<out DbMethod>
+        val idx = vp.index
+        if (idx < 0) {
+            // We're not extracting this and this@TYPE parameters of functions:
+            logger.warn(Severity.ErrorSevere, "Unexpected negative index for parameter")
+        }
+        return "@\"params;{$parentId};$idx\""
+    }
+
+    fun useValueParameter(vp: IrValueParameter): Label<out DbParam> =
+        tw.getLabelFor(getValueParameterLabel(vp))
+
+    fun getFieldLabel(p: IrField): String {
+        val parentId = useDeclarationParent(p.parent)
+        return "@\"field;{$parentId};${p.name.asString()}\""
+    }
+
+    fun useField(p: IrField): Label<out DbField> =
+        tw.getLabelFor(getFieldLabel(p))
+
+    fun getPropertyLabel(p: IrProperty): String {
+        val parentId = useDeclarationParent(p.parent)
+        return "@\"property;{$parentId};${p.name.asString()}\""
+    }
+
+    fun useProperty(p: IrProperty): Label<out DbKt_property> =
+        tw.getLabelFor(getPropertyLabel(p))
+
+    private fun getEnumEntryLabel(ee: IrEnumEntry): String {
+        val parentId = useDeclarationParent(ee.parent)
+        return "@\"field;{$parentId};${ee.name.asString()}\""
+    }
+
+    fun useEnumEntry(ee: IrEnumEntry): Label<out DbField> =
+        tw.getLabelFor(getEnumEntryLabel(ee))
+
+    private fun getTypeAliasLabel(ta: IrTypeAlias): String {
+        val parentId = useDeclarationParent(ta.parent)
+        return "@\"type_alias;{$parentId};${ta.name.asString()}\""
+    }
+
+    fun useTypeAlias(ta: IrTypeAlias): Label<out DbKt_type_alias> =
+        tw.getLabelFor(getTypeAliasLabel(ta))
+
+    fun useVariable(v: IrVariable): Label<out DbLocalvar> {
+        return tw.getVariableLabelFor<DbLocalvar>(v)
+    }
+
+    fun withQuestionMark(t: IrType, hasQuestionMark: Boolean) = if(hasQuestionMark) t.makeNullable() else t.makeNotNull()
+
+}

java/kotlin-extractor/src/main/kotlin/Label.kt

@@ -0,0 +1,29 @@
+package com.github.codeql
+
+import java.io.PrintWriter
+import java.io.StringWriter
+
+interface Label<T>
+
+class IntLabel<T>(val name: Int): Label<T> {
+    override fun toString(): String = "#$name"
+}
+
+class StringLabel<T>(val name: String): Label<T> {
+    override fun toString(): String = "#$name"
+}
+
+class StarLabel<T>: Label<T> {
+    override fun toString(): String = "*"
+}
+
+fun <T> fakeLabel(): Label<T> {
+    if (false) {
+        println("Fake label")
+    } else {
+        val sw = StringWriter()
+        Exception().printStackTrace(PrintWriter(sw))
+        println("Fake label from:\n$sw")
+    }
+    return IntLabel(0)
+}

java/kotlin-extractor/src/main/kotlin/PrimitiveTypeInfo.kt

@@ -0,0 +1,31 @@
+package com.github.codeql
+
+import org.jetbrains.kotlin.ir.types.IdSignatureValues
+
+data class PrimitiveTypeInfo(
+    val primitiveName: String?,
+    val javaPackageName: String, val javaClassName: String,
+    val kotlinPackageName: String, val kotlinClassName: String
+)
+
+val primitiveTypeMapping = mapOf(
+    IdSignatureValues._byte to PrimitiveTypeInfo("byte", "java.lang", "Byte", "kotlin", "Byte"),
+    IdSignatureValues._short to PrimitiveTypeInfo("short", "java.lang", "Short", "kotlin", "Short"),
+    IdSignatureValues._int to PrimitiveTypeInfo("int", "java.lang", "Integer", "kotlin", "Int"),
+    IdSignatureValues._long to PrimitiveTypeInfo("long", "java.lang", "Long", "kotlin", "Long"),
+
+    IdSignatureValues.uByte to PrimitiveTypeInfo("byte", "kotlin", "UByte", "kotlin", "UByte"),
+    IdSignatureValues.uShort to PrimitiveTypeInfo("short", "kotlin", "UShort", "kotlin", "UShort"),
+    IdSignatureValues.uInt to PrimitiveTypeInfo("int", "kotlin", "UInt", "kotlin", "UInt"),
+    IdSignatureValues.uLong to PrimitiveTypeInfo("long", "kotlin", "ULong", "kotlin", "ULong"),
+
+    IdSignatureValues._double to PrimitiveTypeInfo("double", "java.lang", "Double", "kotlin", "Double"),
+    IdSignatureValues._float to PrimitiveTypeInfo("float", "java.lang", "Float", "kotlin", "Float"),
+
+    IdSignatureValues._boolean to PrimitiveTypeInfo("boolean", "java.lang", "Boolean", "kotlin", "Boolean"),
+
+    IdSignatureValues._char to PrimitiveTypeInfo("char", "java.lang", "Character", "kotlin", "Char"),
+
+    IdSignatureValues.unit to PrimitiveTypeInfo("void", "java.lang", "Void", "kotlin", "Nothing"), // TODO: Is this right?
+    IdSignatureValues.nothing to PrimitiveTypeInfo(null, "java.lang", "Void", "kotlin", "Nothing"), // TODO: Is this right?
+)