Merge branch 'main' into redsun82/swift-ios

MODULE.bazel

@@ -218,6 +218,7 @@ use_repo(
     "kotlin-compiler-2.0.0-RC1",
     "kotlin-compiler-2.0.20-Beta2",
     "kotlin-compiler-2.1.0-Beta1",
+    "kotlin-compiler-2.1.20-Beta1",
     "kotlin-compiler-embeddable-1.5.0",
     "kotlin-compiler-embeddable-1.5.10",
     "kotlin-compiler-embeddable-1.5.20",
@@ -232,6 +233,7 @@ use_repo(
     "kotlin-compiler-embeddable-2.0.0-RC1",
     "kotlin-compiler-embeddable-2.0.20-Beta2",
     "kotlin-compiler-embeddable-2.1.0-Beta1",
+    "kotlin-compiler-embeddable-2.1.20-Beta1",
     "kotlin-stdlib-1.5.0",
     "kotlin-stdlib-1.5.10",
     "kotlin-stdlib-1.5.20",
@@ -246,6 +248,7 @@ use_repo(
     "kotlin-stdlib-2.0.0-RC1",
     "kotlin-stdlib-2.0.20-Beta2",
     "kotlin-stdlib-2.1.0-Beta1",
+    "kotlin-stdlib-2.1.20-Beta1",
 )
 
 go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")

docs/codeql/reusables/supported-versions-compilers.rst

@@ -20,7 +20,7 @@
    Java,"Java 7 to 22 [5]_","javac (OpenJDK and Oracle JDK),
 
    Eclipse compiler for Java (ECJ) [6]_",``.java``
-   Kotlin,"Kotlin 1.5.0 to 2.1.0\ *x*","kotlinc",``.kt``
+   Kotlin,"Kotlin 1.5.0 to 2.1.2\ *x*","kotlinc",``.kt``
    JavaScript,ECMAScript 2022 or lower,Not applicable,"``.js``, ``.jsx``, ``.mjs``, ``.es``, ``.es6``, ``.htm``, ``.html``, ``.xhtm``, ``.xhtml``, ``.vue``, ``.hbs``, ``.ejs``, ``.njk``, ``.json``, ``.yaml``, ``.yml``, ``.raml``, ``.xml`` [7]_"
    Python [8]_,"2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13",Not applicable,``.py``
    Ruby [9]_,"up to 3.3",Not applicable,"``.rb``, ``.erb``, ``.gemspec``, ``Gemfile``"

go/ql/lib/change-notes/2024-12-16-any-content-readers.md

@@ -0,0 +1,5 @@
+---
+category: minorAnalysis
+---
+* By implementing `ImplicitFieldReadNode` it is now possible to declare a dataflow node that reads any content (fields, array members, map keys and values). For example, this is appropriate for modelling a serialization method that flattens a potentially deep data structure into a string or byte array.
+* The `Template.Execute[Template]` methods of the `text/template` package now correctly convey taint from any nested fields to their result. This may produce more results from any taint-tracking query when the `text/template` package is in use.

go/ql/lib/ext/text.template.model.yml

@@ -7,5 +7,5 @@ extensions:
       - ["text/template", "", False, "HTMLEscapeString", "", "", "Argument[0]", "ReturnValue", "taint", "manual"]
       - ["text/template", "", False, "JSEscape", "", "", "Argument[1]", "Argument[0]", "taint", "manual"]
       - ["text/template", "", False, "JSEscapeString", "", "", "Argument[0]", "ReturnValue", "taint", "manual"]
-      - ["text/template", "Template", True, "Execute", "", "", "Argument[1]", "Argument[0]", "taint", "manual"]
+#      - ["text/template", "Template", True, "Execute", "", "", "Argument[1]", "Argument[0]", "taint", "manual"] # Implemented in QL to provide an arbitrary content read from the input.
-      - ["text/template", "Template", True, "ExecuteTemplate", "", "", "Argument[2]", "Argument[0]", "taint", "manual"]
+#      - ["text/template", "Template", True, "ExecuteTemplate", "", "", "Argument[2]", "Argument[0]", "taint", "manual"] # Implemented in QL to provide an arbitrary content read from the input.

go/ql/lib/semmle/go/dataflow/internal/DataFlowPrivate.qll

@@ -143,7 +143,8 @@ predicate jumpStep(Node n1, Node n2) {
  * Thus, `node2` references an object with a content `x` that contains the
  * value of `node1`.
  */
-predicate storeStep(Node node1, ContentSet c, Node node2) {
+predicate storeStep(Node node1, ContentSet cs, Node node2) {
+  exists(Content c | cs.asOneContent() = c |
     // a write `(*p).f = rhs` is modeled as two store steps: `rhs` is flows into field `f` of `(*p)`,
     // which in turn flows into the pointer content of `p`
     exists(Write w, Field f, DataFlow::Node base, DataFlow::Node rhs | w.writesField(base, f, rhs) |
@@ -159,10 +160,11 @@ predicate storeStep(Node node1, ContentSet c, Node node2) {
     node1 = node2.(AddressOperationNode).getOperand() and
     c = any(DataFlow::PointerContent pc | pc.getPointerType() = node2.getType())
     or
-  FlowSummaryImpl::Private::Steps::summaryStoreStep(node1.(FlowSummaryNode).getSummaryNode(), c,
-    node2.(FlowSummaryNode).getSummaryNode())
-  or
     containerStoreStep(node1, node2, c)
+  )
+  or
+  FlowSummaryImpl::Private::Steps::summaryStoreStep(node1.(FlowSummaryNode).getSummaryNode(), cs,
+    node2.(FlowSummaryNode).getSummaryNode())
 }
 
 /**
@@ -170,7 +172,8 @@ predicate storeStep(Node node1, ContentSet c, Node node2) {
  * Thus, `node1` references an object with a content `c` whose value ends up in
  * `node2`.
  */
-predicate readStep(Node node1, ContentSet c, Node node2) {
+predicate readStep(Node node1, ContentSet cs, Node node2) {
+  exists(Content c | cs.asOneContent() = c |
     node1 = node2.(PointerDereferenceNode).getOperand() and
     c = any(DataFlow::PointerContent pc | pc.getPointerType() = node1.getType())
     or
@@ -180,10 +183,15 @@ predicate readStep(Node node1, ContentSet c, Node node2) {
       c = any(DataFlow::FieldContent fc | fc.getField() = read.getField())
     )
     or
-  FlowSummaryImpl::Private::Steps::summaryReadStep(node1.(FlowSummaryNode).getSummaryNode(), c,
+    containerReadStep(node1, node2, c)
+  )
+  or
+  FlowSummaryImpl::Private::Steps::summaryReadStep(node1.(FlowSummaryNode).getSummaryNode(), cs,
     node2.(FlowSummaryNode).getSummaryNode())
   or
-  containerReadStep(node1, node2, c)
+  any(ImplicitFieldReadNode ifrn).shouldImplicitlyReadAllFields(node1) and
+  cs.isUniversalContent() and
+  node1 = node2
 }
 
 /**

go/ql/lib/semmle/go/dataflow/internal/DataFlowUtil.qll

@@ -7,6 +7,7 @@ private import semmle.go.dataflow.FunctionInputsAndOutputs
 private import semmle.go.dataflow.ExternalFlow
 private import DataFlowPrivate
 private import FlowSummaryImpl as FlowSummaryImpl
+private import codeql.util.Unit
 import DataFlowNodes::Public
 
 /**
@@ -50,6 +51,18 @@ abstract class FunctionModel extends Function {
   }
 }
 
+/**
+ * A unit class for adding nodes that should implicitly read from all nested content.
+ *
+ * For example, this might be appropriate for the argument to a method that serializes a struct.
+ */
+class ImplicitFieldReadNode extends Unit {
+  /**
+   * Holds if the node `n` should implicitly read from all nested content in a taint-tracking context.
+   */
+  abstract predicate shouldImplicitlyReadAllFields(DataFlow::Node n);
+}
+
 /**
  * Gets the `Node` corresponding to `insn`.
  */
@@ -169,6 +182,11 @@ class Content extends TContent {
   ) {
     filepath = "" and startline = 0 and startcolumn = 0 and endline = 0 and endcolumn = 0
   }
+
+  /**
+   * Gets the `ContentSet` contaning only this content.
+   */
+  ContentSet asContentSet() { result.asOneContent() = this }
 }
 
 /** A reference through a field. */
@@ -236,21 +254,33 @@ class SyntheticFieldContent extends Content, TSyntheticFieldContent {
   override string toString() { result = s.toString() }
 }
 
+private newtype TContentSet =
+  TOneContent(Content c) or
+  TAllContent()
+
 /**
  * An entity that represents a set of `Content`s.
  *
  * The set may be interpreted differently depending on whether it is
  * stored into (`getAStoreContent`) or read from (`getAReadContent`).
  */
-class ContentSet instanceof Content {
+class ContentSet instanceof TContentSet {
   /** Gets a content that may be stored into when storing into this set. */
-  Content getAStoreContent() { result = this }
+  Content getAStoreContent() { this = TOneContent(result) }
 
   /** Gets a content that may be read from when reading from this set. */
-  Content getAReadContent() { result = this }
+  Content getAReadContent() {
+    this = TOneContent(result)
+    or
+    this = TAllContent() and exists(result)
+  }
 
   /** Gets a textual representation of this content set. */
-  string toString() { result = super.toString() }
+  string toString() {
+    exists(Content c | this = TOneContent(c) | result = c.toString())
+    or
+    this = TAllContent() and result = "all content"
+  }
 
   /**
    * Holds if this element is at the specified location.
@@ -262,8 +292,27 @@ class ContentSet instanceof Content {
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
   ) {
-    super.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+    exists(Content c | this = TOneContent(c) |
+      c.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+    )
+    or
+    this = TAllContent() and
+    filepath = "" and
+    startline = 0 and
+    startcolumn = 0 and
+    endline = 0 and
+    endcolumn = 0
   }
+
+  /**
+   * If this is a singleton content set, returns the content.
+   */
+  Content asOneContent() { this = TOneContent(result) }
+
+  /**
+   * Holds if this is a universal content set.
+   */
+  predicate isUniversalContent() { this = TAllContent() }
 }
 
 /**

go/ql/lib/semmle/go/dataflow/internal/FlowSummaryImpl.qll

@@ -61,26 +61,28 @@ module Input implements InputSig<Location, DataFlowImplSpecific::GoDataFlow> {
   }
 
   string encodeContent(ContentSet cs, string arg) {
+    exists(Content c | cs.asOneContent() = c |
       exists(Field f, string package, string className, string fieldName |
-      f = cs.(FieldContent).getField() and
+        f = c.(FieldContent).getField() and
         f.hasQualifiedName(package, className, fieldName) and
         result = "Field" and
         arg = package + "." + className + "." + fieldName
       )
       or
       exists(SyntheticField f |
-      f = cs.(SyntheticFieldContent).getField() and result = "SyntheticField" and arg = f
+        f = c.(SyntheticFieldContent).getField() and result = "SyntheticField" and arg = f
       )
       or
-    cs instanceof ArrayContent and result = "ArrayElement" and arg = ""
+      c instanceof ArrayContent and result = "ArrayElement" and arg = ""
       or
-    cs instanceof CollectionContent and result = "Element" and arg = ""
+      c instanceof CollectionContent and result = "Element" and arg = ""
       or
-    cs instanceof MapKeyContent and result = "MapKey" and arg = ""
+      c instanceof MapKeyContent and result = "MapKey" and arg = ""
       or
-    cs instanceof MapValueContent and result = "MapValue" and arg = ""
+      c instanceof MapValueContent and result = "MapValue" and arg = ""
       or
-    cs instanceof PointerContent and result = "Dereference" and arg = ""
+      c instanceof PointerContent and result = "Dereference" and arg = ""
+    )
   }
 
   bindingset[token]
@@ -523,7 +525,9 @@ module Private {
     SummaryComponent qualifier() { result = argument(-1) }
 
     /** Gets a summary component for field `f`. */
-    SummaryComponent field(Field f) { result = content(any(FieldContent c | c.getField() = f)) }
+    SummaryComponent field(Field f) {
+      result = content(any(FieldContent c | c.getField() = f).asContentSet())
+    }
 
     /** Gets a summary component that represents the return value of a call. */
     SummaryComponent return() { result = SC::return(_) }

go/ql/lib/semmle/go/dataflow/internal/TaintTrackingUtil.qll

@@ -47,10 +47,11 @@ private Type getElementType(Type containerType) {
  * of `c` at sinks and inputs to additional taint steps.
  */
 bindingset[node]
-predicate defaultImplicitTaintRead(DataFlow::Node node, DataFlow::ContentSet c) {
+predicate defaultImplicitTaintRead(DataFlow::Node node, DataFlow::ContentSet cs) {
-  exists(Type containerType |
+  exists(Type containerType, DataFlow::Content c |
     node instanceof DataFlow::ArgumentNode and
-    getElementType*(node.getType()) = containerType
+    getElementType*(node.getType()) = containerType and
+    cs.asOneContent() = c
   |
     containerType instanceof ArrayType and
     c instanceof DataFlow::ArrayContent
@@ -142,7 +143,7 @@ predicate elementWriteStep(DataFlow::Node pred, DataFlow::Node succ) {
   any(DataFlow::Write w).writesElement(succ.(DataFlow::PostUpdateNode).getPreUpdateNode(), _, pred)
   or
   FlowSummaryImpl::Private::Steps::summaryStoreStep(pred.(DataFlowPrivate::FlowSummaryNode)
-        .getSummaryNode(), any(DataFlow::Content c | c instanceof DataFlow::ArrayContent),
+        .getSummaryNode(), any(DataFlow::ArrayContent ac).asContentSet(),
     succ.(DataFlowPrivate::FlowSummaryNode).getSummaryNode())
 }
 

go/ql/lib/semmle/go/frameworks/stdlib/NetHttp.qll

@@ -122,7 +122,7 @@ module NetHttp {
         |
           lastParamIndex = call.getCall().getCalleeType().getNumParameter() - 1 and
           varArgsSliceArgument = SummaryComponentStack::argument(lastParamIndex) and
-          arrayContentSC = SummaryComponent::content(arrayContent) and
+          arrayContentSC = SummaryComponent::content(arrayContent.asContentSet()) and
           stack = SummaryComponentStack::push(arrayContentSC, varArgsSliceArgument)
         )
       else stack = SummaryComponentStack::argument(n)

go/ql/lib/semmle/go/frameworks/stdlib/TextTemplate.qll

@@ -67,4 +67,45 @@ module TextTemplate {
       input = inp and output = outp
     }
   }
+
+  private class ExecuteTemplateMethod extends Method {
+    int inputArg;
+
+    ExecuteTemplateMethod() {
+      exists(string name |
+        this.hasQualifiedName("text/template", "Template", name) and
+        (
+          name = "Execute" and inputArg = 1
+          or
+          name = "ExecuteTemplate" and inputArg = 2
+        )
+      )
+    }
+
+    int getInputArgIdx() { result = inputArg }
+  }
+
+  private class ExecuteTemplateFieldReader extends DataFlow::ImplicitFieldReadNode {
+    override predicate shouldImplicitlyReadAllFields(DataFlow::Node n) {
+      exists(ExecuteTemplateMethod m, DataFlow::MethodCallNode cn |
+        cn.getTarget() = m and
+        n = cn.getArgument(m.getInputArgIdx())
+      )
+    }
+  }
+
+  private class ExecuteTemplateFunctionModels extends TaintTracking::FunctionModel,
+    ExecuteTemplateMethod
+  {
+    FunctionInput inp;
+    FunctionOutput outp;
+
+    ExecuteTemplateFunctionModels() {
+      inp.isParameter(this.getInputArgIdx()) and outp.isParameter(0)
+    }
+
+    override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
+      input = inp and output = outp
+    }
+  }
 }

go/ql/test/library-tests/semmle/go/frameworks/serialization/test.ql

@@ -0,0 +1,9 @@
+import go
+import utils.test.InlineFlowTest
+
+string getArgString(DataFlow::Node src, DataFlow::Node sink) {
+  exists(sink) and
+  result = src.(DataFlow::CallNode).getArgument(0).getExactValue()
+}
+
+import TaintFlowTestArgString<DefaultFlowConfig, getArgString/2>

go/ql/test/library-tests/semmle/go/frameworks/serialization/texttemplate.go

@@ -0,0 +1,60 @@
+package xyz
+
+import (
+	"bytes"
+	"text/template"
+)
+
+type Inner1 struct {
+	Data string
+}
+
+type Inner2 struct {
+	Data string
+}
+
+type Inner3 struct {
+	Data string
+}
+
+type HasInner3Slice struct {
+	Slice []Inner3
+}
+
+type Outer struct {
+	SliceField []Inner1
+	PtrField   *Inner2
+	MapField   map[string]Inner3
+	DeepField  HasInner3Slice
+}
+
+func source(n int) string { return "dummy" }
+func sink(arg any)        {}
+
+func test() {
+
+	source1 := source(1)
+	source2 := source(2)
+	source3 := source(3)
+	source4 := source(4)
+
+	toSerialize := Outer{[]Inner1{{source1}}, &Inner2{source2}, map[string]Inner3{"key": {source3}},
+		HasInner3Slice{[]Inner3{{source4}}}}
+	buff1 := new(bytes.Buffer)
+	buff2 := new(bytes.Buffer)
+	bytes1 := make([]byte, 10)
+	bytes2 := make([]byte, 10)
+
+	tmpl, _ := template.New("test").Parse("Template text goes here (irrelevant for test)")
+	tmpl.ExecuteTemplate(buff1, "test", toSerialize)
+	buff1.Read(bytes1)
+	sink(bytes1) // $ hasTaintFlow=1 hasTaintFlow=2 hasTaintFlow=3 hasTaintFlow=4
+
+	// Read `buff2` via an `any`-typed variable, to ensure the static type of the argument to tmpl.Execute makes no difference to the result
+	var toSerializeAsAny any
+	toSerializeAsAny = toSerialize
+	tmpl.Execute(buff2, toSerializeAsAny)
+	buff2.Read(bytes2)
+	sink(bytes2) // $ hasTaintFlow=1 hasTaintFlow=2 hasTaintFlow=3 hasTaintFlow=4
+
+}

java/downgrades/38d02c063878000356a3e5db49d5a6a8f38efe24/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add databaseMetadata relation
+compatibility: full
+databaseMetadata.rel: delete

java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt

@@ -25,7 +25,19 @@ import org.jetbrains.kotlin.ir.declarations.lazy.IrLazyFunction
 import org.jetbrains.kotlin.ir.expressions.*
 import org.jetbrains.kotlin.ir.expressions.impl.*
 import org.jetbrains.kotlin.ir.symbols.*
-import org.jetbrains.kotlin.ir.types.*
+import org.jetbrains.kotlin.ir.types.classFqName
+import org.jetbrains.kotlin.ir.types.classifierOrFail
+import org.jetbrains.kotlin.ir.types.classOrNull
+import org.jetbrains.kotlin.ir.types.isAny
+import org.jetbrains.kotlin.ir.types.isNullableAny
+import org.jetbrains.kotlin.ir.types.typeOrNull
+import org.jetbrains.kotlin.ir.types.typeWith
+import org.jetbrains.kotlin.ir.types.typeWithArguments
+import org.jetbrains.kotlin.ir.types.IrSimpleType
+import org.jetbrains.kotlin.ir.types.IrStarProjection
+import org.jetbrains.kotlin.ir.types.IrType
+import org.jetbrains.kotlin.ir.types.IrTypeArgument
+import org.jetbrains.kotlin.ir.types.IrTypeProjection
 import org.jetbrains.kotlin.ir.types.impl.makeTypeProjection
 import org.jetbrains.kotlin.ir.util.*
 import org.jetbrains.kotlin.load.java.JvmAnnotationNames
@@ -2293,7 +2305,7 @@ open class KotlinFileExtractor(
             // synthesised and inherit the annotation from the delegate (which given it has
             // @NotNull, is likely written in Java)
             JvmAnnotationNames.JETBRAINS_NOT_NULL_ANNOTATION.takeUnless {
-                t.isNullable() ||
+                t.isNullableCodeQL() ||
                     primitiveTypeMapping.getPrimitiveInfo(t) != null ||
                     hasExistingAnnotation(it)
             }
@@ -3975,7 +3987,7 @@ open class KotlinFileExtractor(
                 target.parent
             } else {
                 val st = extensionReceiverParameter.type as? IrSimpleType
-                if (isNullable != null && st?.isNullable() != isNullable) {
+                if (isNullable != null && st?.isNullableCodeQL() != isNullable) {
                     verboseln("Nullablility of type didn't match")
                     return false
                 }
@@ -4621,9 +4633,9 @@ open class KotlinFileExtractor(
                     val isPrimitiveArrayCreation = !isBuiltinCallKotlin(c, "arrayOf")
                     val elementType =
                         if (isPrimitiveArrayCreation) {
-                            c.type.getArrayElementType(pluginContext.irBuiltIns)
+                            c.type.getArrayElementTypeCodeQL(pluginContext.irBuiltIns)
                         } else {
-                            // TODO: is there any reason not to always use getArrayElementType?
+                            // TODO: is there any reason not to always use getArrayElementTypeCodeQL?
                             if (c.typeArgumentsCount == 1) {
                                 c.getTypeArgument(0).also {
                                     if (it == null) {

java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt

@@ -12,7 +12,24 @@ import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.expressions.*
 import org.jetbrains.kotlin.ir.symbols.*
-import org.jetbrains.kotlin.ir.types.*
+import org.jetbrains.kotlin.ir.types.addAnnotations
+import org.jetbrains.kotlin.ir.types.classFqName
+import org.jetbrains.kotlin.ir.types.classifierOrNull
+import org.jetbrains.kotlin.ir.types.classOrNull
+import org.jetbrains.kotlin.ir.types.isAny
+import org.jetbrains.kotlin.ir.types.isNullableAny
+import org.jetbrains.kotlin.ir.types.isPrimitiveType
+import org.jetbrains.kotlin.ir.types.makeNullable
+import org.jetbrains.kotlin.ir.types.typeOrNull
+import org.jetbrains.kotlin.ir.types.typeWith
+import org.jetbrains.kotlin.ir.types.typeWithArguments
+import org.jetbrains.kotlin.ir.types.IrDynamicType
+import org.jetbrains.kotlin.ir.types.IrErrorType
+import org.jetbrains.kotlin.ir.types.IrSimpleType
+import org.jetbrains.kotlin.ir.types.IrStarProjection
+import org.jetbrains.kotlin.ir.types.IrType
+import org.jetbrains.kotlin.ir.types.IrTypeArgument
+import org.jetbrains.kotlin.ir.types.IrTypeProjection
 import org.jetbrains.kotlin.ir.types.impl.*
 import org.jetbrains.kotlin.ir.util.*
 import org.jetbrains.kotlin.load.java.BuiltinMethodsWithSpecialGenericSignature
@@ -679,7 +696,7 @@ open class KotlinUsesExtractor(
     private fun getInvariantNullableArrayType(arrayType: IrSimpleType): IrSimpleType =
         if (arrayType.isPrimitiveArray()) arrayType
         else {
-            val componentType = arrayType.getArrayElementType(pluginContext.irBuiltIns)
+            val componentType = arrayType.getArrayElementTypeCodeQL(pluginContext.irBuiltIns)
             val componentTypeBroadened =
                 when (componentType) {
                     is IrSimpleType ->
@@ -690,7 +707,7 @@ open class KotlinUsesExtractor(
             val unchanged =
                 componentType == componentTypeBroadened &&
                     (arrayType.arguments[0] as? IrTypeProjection)?.variance == Variance.INVARIANT &&
-                    componentType.isNullable()
+                    componentType.isNullableCodeQL()
             if (unchanged) arrayType
             else
                 IrSimpleTypeImpl(
@@ -705,7 +722,7 @@ open class KotlinUsesExtractor(
     Kotlin arrays can be broken down as:
 
     isArray(t)
-    |- t.isBoxedArray
+    |- t.isBoxedArrayCodeQL
     |  |- t.isArray()         e.g. Array<Boolean>, Array<Boolean?>
     |  |- t.isNullableArray() e.g. Array<Boolean>?, Array<Boolean?>?
     |- t.isPrimitiveArray()   e.g. BooleanArray
@@ -715,7 +732,7 @@ open class KotlinUsesExtractor(
     Primitive arrays are represented as e.g. boolean[].
     */
 
-    private fun isArray(t: IrType) = t.isBoxedArray || t.isPrimitiveArray()
+    private fun isArray(t: IrType) = t.isBoxedArrayCodeQL || t.isPrimitiveArray()
 
     data class ArrayInfo(
         val elementTypeResults: TypeResults,
@@ -756,7 +773,7 @@ open class KotlinUsesExtractor(
             ) {
                 pluginContext.irBuiltIns.anyType
             } else {
-                t.getArrayElementType(pluginContext.irBuiltIns)
+                t.getArrayElementTypeCodeQL(pluginContext.irBuiltIns)
             }
 
         val recInfo = useArrayType(elementType, t.isPrimitiveArray())
@@ -844,7 +861,7 @@ open class KotlinUsesExtractor(
                 if (
                     (context == TypeContext.RETURN ||
                         (context == TypeContext.OTHER && otherIsPrimitive)) &&
-                        !s.isNullable() &&
+                        !s.isNullableCodeQL() &&
                         getKotlinType(s)?.hasEnhancedNullability() != true &&
                         primitiveName != null
                 ) {
@@ -860,7 +877,7 @@ open class KotlinUsesExtractor(
             val kotlinClassId = useClassInstance(kotlinClass, listOf()).typeResult.id
             val kotlinResult =
                 if (true) TypeResult(fakeKotlinType(), "TODO", "TODO")
-                else if (s.isNullable()) {
+                else if (s.isNullableCodeQL()) {
                     val kotlinSignature =
                         "$kotlinPackageName.$kotlinClassName?" // TODO: Is this right?
                     val kotlinLabel = "@\"kt_type;nullable;$kotlinPackageName.$kotlinClassName\""
@@ -902,21 +919,21 @@ open class KotlinUsesExtractor(
                     return extractErrorType()
                 }
             }
-            (s.isBoxedArray && s.arguments.isNotEmpty()) || s.isPrimitiveArray() -> {
+            (s.isBoxedArrayCodeQL && s.arguments.isNotEmpty()) || s.isPrimitiveArray() -> {
                 val arrayInfo = useArrayType(s, false)
                 return arrayInfo.componentTypeResults
             }
             owner is IrClass -> {
                 val args = if (s.codeQlIsRawType()) null else s.arguments
 
-                return useSimpleTypeClass(owner, args, s.isNullable())
+                return useSimpleTypeClass(owner, args, s.isNullableCodeQL())
             }
             owner is IrTypeParameter -> {
                 val javaResult = useTypeParameter(owner)
                 val aClassId = makeClass("kotlin", "TypeParam") // TODO: Wrong
                 val kotlinResult =
                     if (true) TypeResult(fakeKotlinType(), "TODO", "TODO")
-                    else if (s.isNullable()) {
+                    else if (s.isNullableCodeQL()) {
                         val kotlinSignature = "${javaResult.signature}?" // TODO: Wrong
                         val kotlinLabel = "@\"kt_type;nullable;type_param\"" // TODO: Wrong
                         val kotlinId: Label<DbKt_nullable_type> =
@@ -1200,7 +1217,7 @@ open class KotlinUsesExtractor(
         }
 
     private fun extendsAdditionAllowed(t: IrType) =
-        if (t.isBoxedArray) {
+        if (t.isBoxedArrayCodeQL) {
             if (t is IrSimpleType) {
                 arrayExtendsAdditionAllowed(t)
             } else {
@@ -1493,7 +1510,7 @@ open class KotlinUsesExtractor(
                     }
                 } else {
                     t.classOrNull?.let { tCls ->
-                        if (t.isBoxedArray) {
+                        if (t.isBoxedArrayCodeQL) {
                             (t.arguments.singleOrNull() as? IrTypeProjection)?.let { elementTypeArg
                                 ->
                                 val elementType = elementTypeArg.type
@@ -1506,7 +1523,7 @@ open class KotlinUsesExtractor(
                                         )
                                     return tCls
                                         .typeWithArguments(listOf(newArg))
-                                        .codeQlWithHasQuestionMark(t.isNullable())
+                                        .codeQlWithHasQuestionMark(t.isNullableCodeQL())
                                 }
                             }
                         }
@@ -2086,12 +2103,12 @@ open class KotlinUsesExtractor(
             }
 
             if (owner is IrClass) {
-                if (t.isBoxedArray) {
+                if (t.isBoxedArrayCodeQL) {
-                    val elementType = t.getArrayElementType(pluginContext.irBuiltIns)
+                    val elementType = t.getArrayElementTypeCodeQL(pluginContext.irBuiltIns)
                     val erasedElementType = erase(elementType)
                     return owner
                         .typeWith(erasedElementType)
-                        .codeQlWithHasQuestionMark(t.isNullable())
+                        .codeQlWithHasQuestionMark(t.isNullableCodeQL())
                 }
 
                 return if (t.arguments.isNotEmpty())

java/kotlin-extractor/src/main/kotlin/utils/ClassNames.kt

@@ -5,7 +5,7 @@ import com.github.codeql.utils.versions.*
 import com.intellij.openapi.vfs.StandardFileSystems
 import com.intellij.openapi.vfs.VirtualFile
 import org.jetbrains.kotlin.builtins.jvm.JavaToKotlinClassMap
-import org.jetbrains.kotlin.fir.java.JavaBinarySourceElement
+import org.jetbrains.kotlin.fir.java.VirtualFileBasedSourceElement
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable
@@ -89,8 +89,8 @@ fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? {
                 is BinaryJavaClass -> return element.virtualFile
             }
         }
-        is JavaBinarySourceElement -> {
+        is VirtualFileBasedSourceElement -> {
-            return cSource.javaClass.virtualFile
+            return cSource.virtualFile
         }
         is KotlinJvmBinarySourceElement -> {
             val binaryClass = cSource.binaryClass

java/kotlin-extractor/src/main/kotlin/utils/TypeSubstitution.kt

@@ -5,6 +5,7 @@ import com.github.codeql.Logger
 import com.github.codeql.getJavaEquivalentClassId
 import com.github.codeql.utils.versions.codeQlWithHasQuestionMark
 import com.github.codeql.utils.versions.createImplicitParameterDeclarationWithWrappedDescriptor
+import com.github.codeql.utils.versions.*
 import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
 import org.jetbrains.kotlin.descriptors.ClassKind
 import org.jetbrains.kotlin.ir.builders.declarations.addConstructor
@@ -18,15 +19,18 @@ import org.jetbrains.kotlin.ir.expressions.IrConstructorCall
 import org.jetbrains.kotlin.ir.expressions.impl.*
 import org.jetbrains.kotlin.ir.symbols.IrTypeParameterSymbol
 import org.jetbrains.kotlin.ir.symbols.impl.DescriptorlessExternalPackageFragmentSymbol
-import org.jetbrains.kotlin.ir.types.*
+import org.jetbrains.kotlin.ir.types.addAnnotations
+import org.jetbrains.kotlin.ir.types.classifierOrNull
+import org.jetbrains.kotlin.ir.types.typeWith
+import org.jetbrains.kotlin.ir.types.IrSimpleType
+import org.jetbrains.kotlin.ir.types.IrStarProjection
+import org.jetbrains.kotlin.ir.types.IrType
+import org.jetbrains.kotlin.ir.types.IrTypeArgument
+import org.jetbrains.kotlin.ir.types.IrTypeProjection
 import org.jetbrains.kotlin.ir.types.impl.IrSimpleTypeImpl
 import org.jetbrains.kotlin.ir.types.impl.IrStarProjectionImpl
 import org.jetbrains.kotlin.ir.types.impl.makeTypeProjection
-import org.jetbrains.kotlin.ir.util.classId
+import org.jetbrains.kotlin.ir.util.*
-import org.jetbrains.kotlin.ir.util.constructedClassType
-import org.jetbrains.kotlin.ir.util.constructors
-import org.jetbrains.kotlin.ir.util.kotlinFqName
-import org.jetbrains.kotlin.ir.util.parents
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.types.Variance
@@ -57,7 +61,7 @@ private fun IrSimpleType.substituteTypeArguments(
             }
         }
 
-    return IrSimpleTypeImpl(classifier, isNullable(), newArguments, annotations)
+    return IrSimpleTypeImpl(classifier, isNullableCodeQL(), newArguments, annotations)
 }
 
 /**
@@ -96,7 +100,7 @@ private fun subProjectedType(
             else {
                 val newProjectedType =
                     substitutedTypeArg.type.let {
-                        if (t.isNullable()) it.codeQlWithHasQuestionMark(true) else it
+                        if (t.isNullableCodeQL()) it.codeQlWithHasQuestionMark(true) else it
                     }
                 val newVariance = combineVariance(outerVariance, substitutedTypeArg.variance)
                 makeTypeProjection(newProjectedType, newVariance)
@@ -113,7 +117,7 @@ private fun IrTypeArgument.upperBound(context: IrPluginContext) =
             when (this.variance) {
                 Variance.INVARIANT -> this.type
                 Variance.IN_VARIANCE ->
-                    if (this.type.isNullable()) context.irBuiltIns.anyNType
+                    if (this.type.isNullableCodeQL()) context.irBuiltIns.anyNType
                     else context.irBuiltIns.anyType
                 Variance.OUT_VARIANCE -> this.type
             }
@@ -128,7 +132,7 @@ private fun IrTypeArgument.lowerBound(context: IrPluginContext) =
                 Variance.INVARIANT -> this.type
                 Variance.IN_VARIANCE -> this.type
                 Variance.OUT_VARIANCE ->
-                    if (this.type.isNullable()) context.irBuiltIns.nothingNType
+                    if (this.type.isNullableCodeQL()) context.irBuiltIns.nothingNType
                     else context.irBuiltIns.nothingType
             }
         else -> context.irBuiltIns.nothingType
@@ -211,7 +215,7 @@ fun IrTypeArgument.withQuestionMark(b: Boolean): IrTypeArgument =
             this.type.let {
                 when (it) {
                     is IrSimpleType ->
-                        if (it.isNullable() == b) this
+                        if (it.isNullableCodeQL() == b) this
                         else makeTypeProjection(it.codeQlWithHasQuestionMark(b), this.variance)
                     else -> this
                 }

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/IrBuiltIns.kt

@@ -0,0 +1,5 @@
+package com.github.codeql.utils.versions
+
+import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
+
+typealias IrBuiltIns = IrBuiltIns

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/VirtualFileBasedSourceElement.kt

@@ -1,5 +1,6 @@
 package org.jetbrains.kotlin.fir.java
 
+import com.intellij.openapi.vfs.VirtualFile
 import org.jetbrains.kotlin.descriptors.SourceElement
 import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
 
@@ -7,5 +8,6 @@ import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
 We need this class to exist, but the compiler will never give us an
 instance of it.
 */
-abstract class JavaBinarySourceElement private constructor(val javaClass: BinaryJavaClass) :
+abstract class VirtualFileBasedSourceElement private constructor(val javaClass: BinaryJavaClass) : SourceElement {
-    SourceElement {}
+    abstract val virtualFile: VirtualFile
+}

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_0/typeUtils.kt

@@ -0,0 +1,11 @@
+package com.github.codeql.utils.versions
+
+import org.jetbrains.kotlin.ir.types.*
+
+fun IrType.isNullableCodeQL(): Boolean =
+    this.isNullable()
+
+val IrType.isBoxedArrayCodeQL: Boolean by IrType::isBoxedArray
+
+fun IrType.getArrayElementTypeCodeQL(irBuiltIns: IrBuiltIns): IrType =
+   this.getArrayElementType(irBuiltIns)

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_6_0/IrBuiltIns.kt

@@ -0,0 +1,5 @@
+package com.github.codeql.utils.versions
+
+import org.jetbrains.kotlin.ir.IrBuiltIns
+
+typealias IrBuiltIns = org.jetbrains.kotlin.ir.IrBuiltIns

java/kotlin-extractor/src/main/kotlin/utils/versions/v_2_0_0-RC1/JavaBinarySourceElement.kt

@@ -1,3 +0,0 @@
-/*
-The compiler provides this class, so we don't have to do anything.
-*/

java/kotlin-extractor/src/main/kotlin/utils/versions/v_2_0_0-RC1/VirtualFileBasedSourceElement.kt

@@ -0,0 +1,3 @@
+package org.jetbrains.kotlin.fir.java
+
+typealias VirtualFileBasedSourceElement = JavaBinarySourceElement

java/kotlin-extractor/src/main/kotlin/utils/versions/v_2_1_20-Beta1/VirtualFileBasedSourceElement.kt

@@ -0,0 +1 @@
+// Nothing to do

java/kotlin-extractor/src/main/kotlin/utils/versions/v_2_1_20-Beta1/typeUtils.kt

@@ -0,0 +1,12 @@
+package com.github.codeql.utils.versions
+
+import org.jetbrains.kotlin.ir.types.IrType
+import org.jetbrains.kotlin.ir.util.*
+
+fun IrType.isNullableCodeQL(): Boolean =
+    this.isNullable()
+
+val IrType.isBoxedArrayCodeQL: Boolean by IrType::isBoxedArray
+
+fun IrType.getArrayElementTypeCodeQL(irBuiltIns: IrBuiltIns): IrType =
+   this.getArrayElementType(irBuiltIns)

java/kotlin-extractor/versions.bzl

@@ -14,6 +14,7 @@ VERSIONS = [
     "2.0.0-RC1",
     "2.0.20-Beta2",
     "2.1.0-Beta1",
+    "2.1.20-Beta1",
 ]
 
 def _version_to_tuple(v):

java/ql/integration-tests/kotlin/all-platforms/diagnostics/kotlin-version-too-new/diagnostics.expected

@@ -1,5 +1,5 @@
 {
-  "markdownMessage": "The Kotlin version installed (`999.999.999`) is too recent for this version of CodeQL. Install a version lower than 2.1.10.",
+  "markdownMessage": "The Kotlin version installed (`999.999.999`) is too recent for this version of CodeQL. Install a version lower than 2.1.30.",
   "severity": "error",
   "source": {
     "extractorName": "java",

java/ql/lib/config/semmlecode.dbscheme

@@ -206,6 +206,11 @@ sourceLocationPrefix(
   string prefix : string ref
 );
 
+databaseMetadata(
+  string metadataKey : string ref,
+  string value : string ref
+);
+
 /*
  * SMAP
  */

java/ql/lib/config/semmlecode.dbscheme.stats

@@ -3968,6 +3968,21 @@
             </columnsizes>
             <dependencies/>
         </relation>
+        <relation>
+            <name>databaseMetadata</name>
+            <cardinality>1</cardinality>
+            <columnsizes>
+                <e>
+                    <k>metadataKey</k>
+                    <v>1</v>
+                </e>
+                <e>
+                    <k>value</k>
+                    <v>1</v>
+                </e>
+            </columnsizes>
+            <dependencies/>
+        </relation>
         <relation>
             <name>smap_header</name>
             <cardinality>1</cardinality>

java/ql/lib/upgrades/1fd1afa7862b82955785edd29820054ab6c9ec81/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add databaseMetadata relation
+compatibility: full