Legacy support qlpacks: continue using libraryPathDependencies; add a comment noting this is obsolete.

Kotlin: Remove `javaEquivalent` consistency query

.github/labeler.yml

@@ -43,3 +43,11 @@ documentation:
 "QL-for-QL":
   - ql/**/*
   - .github/workflows/ql-for-ql*
+
+# Since these are all shared files that need to be synced, just pick _one_ copy of each.
+"DataFlow Library":
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"

.github/workflows/atm-check-queries-run.yml

@@ -0,0 +1,13 @@
+name: ATM Check Queries Run
+
+# This check is required, therefore we must run it on all PRs, even if only Markdown has changed.
+on:
+  workflow_dispatch:
+
+jobs:
+  hello-world:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: foo
+        run: echo "Hello world"

.github/workflows/swift-qltest.yml

@@ -23,12 +23,23 @@ jobs:
       - uses: ./.github/actions/fetch-codeql
       - name: Check QL formatting
         run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
+  qltest-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version-file: 'swift/.python-version'
+      - name: Test qltest.sh
+        run: |
+          bazel test //swift/tools/test/qltest
   qltest:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os : [ubuntu-20.04, macos-latest]
+        os: [ ubuntu-20.04, macos-latest ]
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql

change-notes/1.20/analysis-javascript.md

@@ -52,7 +52,7 @@
 | Unneeded defensive code | More true positive and fewer false positive results | This query now recognizes additional defensive code patterns. |
 | Unsafe dynamic method access              | Fewer false positive results | This query no longer flags concatenated strings as unsafe method names. |
 | Unused parameter                           | Fewer false positive results | This query no longer flags parameters with leading underscore. |
-| Unused variable, import, function or class | Fewer false positive results | This query now flags fewer variables that are implictly used by JSX elements. It no longer flags variables with a leading underscore and variables in dead code. |
+| Unused variable, import, function or class | Fewer false positive results | This query now flags fewer variables that are implicitly used by JSX elements. It no longer flags variables with a leading underscore and variables in dead code. |
 | Unvalidated dynamic method call           | More true positive results | This query now flags concatenated strings as unvalidated method names in more cases. |
 | Useless assignment to property. | Fewer false positive results | This query now treats assignments with complex right-hand sides correctly. |
 | Useless conditional | Fewer results | Additional defensive coding patterns are now ignored. |

change-notes/1.23/analysis-cpp.md

@@ -19,7 +19,7 @@ The following changes in version 1.23 affect C/C++ analysis in all applications.
 | Hard-coded Japanese era start date in call (`cpp/japanese-era/constructor-or-method-with-exact-era-date`) | Deprecated | This query has been deprecated.  Use the new combined query Hard-coded Japanese era start date (`cpp/japanese-era/exact-era-date`) instead. |
 | Hard-coded Japanese era start date in struct (`cpp/japanese-era/struct-with-exact-era-date`) | Deprecated | This query has been deprecated.  Use the new combined query Hard-coded Japanese era start date (`cpp/japanese-era/exact-era-date`) instead. |
 | Hard-coded Japanese era start date (`cpp/japanese-era/exact-era-date`) | More correct results | This query now checks for the beginning date of the Reiwa era (1st May 2019). |
-| Non-constant format string (`cpp/non-constant-format`) | Fewer false positive results | Fixed false positive results triggrered by mismatching declarations of a formatting function. |
+| Non-constant format string (`cpp/non-constant-format`) | Fewer false positive results | Fixed false positive results triggered by mismatching declarations of a formatting function. |
 | Sign check of bitwise operation (`cpp/bitwise-sign-check`) | Fewer false positive results | Results involving `>=` or `<=` are no longer reported. |
 | Too few arguments to formatting function (`cpp/wrong-number-format-arguments`) | Fewer false positive results | Fixed false positive results triggered by mismatching declarations of a formatting function. |
 | Too many arguments to formatting function (`cpp/too-many-format-arguments`) | Fewer false positive results | Fixed false positive results triggered by mismatching declarations of a formatting function. |

change-notes/1.24/analysis-javascript.md

@@ -91,7 +91,7 @@
 
 ## Changes to libraries
 
-* The predicates `RegExpTerm.getSuccessor` and `RegExpTerm.getPredecessor` have been changed to reflect textual, not operational, matching order. This only makes a difference in lookbehind assertions, which are operationally matched backwards. Previously, `getSuccessor` would mimick this, so in an assertion `(?<=ab)` the term `b` would be considered the predecessor, not the successor, of `a`. Textually, however, `a` is still matched before `b`, and this is the order we now follow.
+* The predicates `RegExpTerm.getSuccessor` and `RegExpTerm.getPredecessor` have been changed to reflect textual, not operational, matching order. This only makes a difference in lookbehind assertions, which are operationally matched backwards. Previously, `getSuccessor` would mimic this, so in an assertion `(?<=ab)` the term `b` would be considered the predecessor, not the successor, of `a`. Textually, however, `a` is still matched before `b`, and this is the order we now follow.
 * An extensible model of the `EventEmitter` pattern has been implemented.
 * Taint-tracking configurations now interact differently with the `data` flow label, which may affect queries
   that combine taint-tracking and flow labels.

cpp/ql/lib/change-notes/2022-10-22-format-literal.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Fixed bugs in the `FormatLiteral` class that were causing `getMaxConvertedLength` and related predicates to return no results when the format literal was `%e`, `%f` or `%g` and an explicit precision was specified.

cpp/ql/lib/experimental/semmle/code/cpp/semantic/analysis/ModulusAnalysis.qll

@@ -4,6 +4,12 @@
  * variable), and `v` is an integer in the range `[0 .. m-1]`.
  */
 
+/*
+ * The main recursion has base cases in both `ssaModulus` (for guarded reads) and `semExprModulus`
+ * (for constant values). The most interesting recursive case is `phiModulusRankStep`, which
+ * handles phi inputs.
+ */
+
 private import ModulusAnalysisSpecific::Private
 private import experimental.semmle.code.cpp.semantic.Semantic
 private import ConstantAnalysis
@@ -162,6 +168,11 @@ private predicate phiModulusInit(SemSsaPhiNode phi, SemBound b, int val, int mod
  */
 pragma[nomagic]
 private predicate phiModulusRankStep(SemSsaPhiNode phi, SemBound b, int val, int mod, int rix) {
+  /*
+   * base case. If any phi input is equal to `b + val` modulo `mod`, that's a potential congruence
+   * class for the phi node.
+   */
+
   rix = 0 and
   phiModulusInit(phi, b, val, mod)
   or
@@ -169,6 +180,12 @@ private predicate phiModulusRankStep(SemSsaPhiNode phi, SemBound b, int val, int
     mod != 1 and
     val = remainder(v1, mod)
   |
+    /*
+     * Recursive case. If `inp` = `b + v2` mod `m2`, we combine that with the preceding potential
+     * congruence class `b + v1` mod `m1`. The result will be the congruence class of `v1` modulo
+     * the greatest common denominator of `m1`, `m2`, and `v1 - v2`.
+     */
+
     exists(int v2, int m2 |
       rankedPhiInput(pragma[only_bind_out](phi), inp, edge, rix) and
       phiModulusRankStep(phi, b, v1, m1, rix - 1) and
@@ -176,6 +193,12 @@ private predicate phiModulusRankStep(SemSsaPhiNode phi, SemBound b, int val, int
       mod = m1.gcd(m2).gcd(v1 - v2)
     )
     or
+    /*
+     * Recursive case. If `inp` = `phi` mod `m2`, we combine that with the preceding potential
+     * congruence class `b + v1` mod `m1`. The result will be a congruence class modulo the greatest
+     * common denominator of `m1` and `m2`.
+     */
+
     exists(int m2 |
       rankedPhiInput(phi, inp, edge, rix) and
       phiModulusRankStep(phi, b, v1, m1, rix - 1) and

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.4.2
+version: 0.4.3-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/commons/Printf.qll

@@ -1125,12 +1125,12 @@ class FormatLiteral extends Literal {
         exists(int dot, int afterdot |
           (if this.getPrecision(n) = 0 then dot = 0 else dot = 1) and
           (
-            (
-              if this.hasExplicitPrecision(n)
-              then afterdot = this.getPrecision(n)
-              else not this.hasImplicitPrecision(n)
-            ) and
-            afterdot = 6
+            if this.hasExplicitPrecision(n)
+            then afterdot = this.getPrecision(n)
+            else (
+              not this.hasImplicitPrecision(n) and
+              afterdot = 6
+            )
           ) and
           len = 1 + 309 + dot + afterdot
         ) and
@@ -1140,12 +1140,12 @@ class FormatLiteral extends Literal {
         exists(int dot, int afterdot |
           (if this.getPrecision(n) = 0 then dot = 0 else dot = 1) and
           (
-            (
-              if this.hasExplicitPrecision(n)
-              then afterdot = this.getPrecision(n)
-              else not this.hasImplicitPrecision(n)
-            ) and
-            afterdot = 6
+            if this.hasExplicitPrecision(n)
+            then afterdot = this.getPrecision(n)
+            else (
+              not this.hasImplicitPrecision(n) and
+              afterdot = 6
+            )
           ) and
           len = 1 + 1 + dot + afterdot + 1 + 1 + 3
         ) and
@@ -1155,12 +1155,12 @@ class FormatLiteral extends Literal {
         exists(int dot, int afterdot |
           (if this.getPrecision(n) = 0 then dot = 0 else dot = 1) and
           (
-            (
-              if this.hasExplicitPrecision(n)
-              then afterdot = this.getPrecision(n)
-              else not this.hasImplicitPrecision(n)
-            ) and
-            afterdot = 6
+            if this.hasExplicitPrecision(n)
+            then afterdot = this.getPrecision(n)
+            else (
+              not this.hasImplicitPrecision(n) and
+              afterdot = 6
+            )
           ) and
           // note: this could be displayed in the style %e or %f;
           //       however %f is only used when 'P > X >= -4'

cpp/ql/src/Best Practices/Likely Errors/CommaBeforeMisleadingIndentation.ql

@@ -16,15 +16,36 @@
 import cpp
 import semmle.code.cpp.commons.Exclusions
 
-/** Gets the sub-expression of 'e' with the earliest-starting Location */
+/**
+ * Gets a child of `e`, including conversions but excluding call arguments.
+ */
+pragma[inline]
+Expr getAChildWithConversions(Expr e) {
+  result.getParentWithConversions() = e and
+  not result = any(Call c).getAnArgument()
+}
+
+/**
+ * Gets the left-most column position of any transitive child of `e` (including
+ * conversions but excluding call arguments).
+ */
+int getCandidateColumn(Expr e) {
+  result = e.getLocation().getStartColumn() or
+  result = getCandidateColumn(getAChildWithConversions(e))
+}
+
+/**
+ * Gets the transitive child of `e` (including conversions but excluding call
+ * arguments) at the left-most column position, preferring less deeply nested
+ * expressions if there is a choice.
+ */
 Expr normalizeExpr(Expr e) {
-  result =
-    min(Expr child |
-      child.getParentWithConversions*() = e.getFullyConverted() and
-      not child.getParentWithConversions*() = any(Call c).getAnArgument()
-    |
-      child order by child.getLocation().getStartColumn(), count(child.getParentWithConversions*())
-    )
+  e.getLocation().getStartColumn() = min(getCandidateColumn(e)) and
+  result = e
+  or
+  not e.getLocation().getStartColumn() = min(getCandidateColumn(e)) and
+  result = normalizeExpr(getAChildWithConversions(e)) and
+  result.getLocation().getStartColumn() = min(getCandidateColumn(e))
 }
 
 predicate isParenthesized(CommaExpr ce) {
@@ -43,8 +64,8 @@ from CommaExpr ce, Expr left, Expr right, Location leftLoc, Location rightLoc
 where
   ce.fromSource() and
   not isFromMacroDefinition(ce) and
-  left = normalizeExpr(ce.getLeftOperand()) and
-  right = normalizeExpr(ce.getRightOperand()) and
+  left = normalizeExpr(ce.getLeftOperand().getFullyConverted()) and
+  right = normalizeExpr(ce.getRightOperand().getFullyConverted()) and
   leftLoc = left.getLocation() and
   rightLoc = right.getLocation() and
   not isParenthesized(ce) and

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.4.2
+version: 0.4.3-dev
 groups: 
   - cpp
   - queries

cpp/ql/test/TestUtilities/InlineExpectationsTest.qll

@@ -137,6 +137,7 @@ abstract class InlineExpectationsTest extends string {
   final predicate hasFailureMessage(FailureLocatable element, string message) {
     exists(ActualResult actualResult |
       actualResult.getTest() = this and
+      actualResult.getTag() = this.getARelevantTag() and
       element = actualResult and
       (
         exists(FalseNegativeExpectation falseNegative |
@@ -150,9 +151,18 @@ abstract class InlineExpectationsTest extends string {
       )
     )
     or
+    exists(ActualResult actualResult |
+      actualResult.getTest() = this and
+      not actualResult.getTag() = this.getARelevantTag() and
+      element = actualResult and
+      message =
+        "Tag mismatch: Actual result with tag '" + actualResult.getTag() +
+          "' that is not part of getARelevantTag()"
+    )
+    or
     exists(ValidExpectation expectation |
       not exists(ActualResult actualResult | expectation.matchesActualResult(actualResult)) and
-      expectation.getTag() = getARelevantTag() and
+      expectation.getTag() = this.getARelevantTag() and
       element = expectation and
       (
         expectation instanceof GoodExpectation and

cpp/ql/test/library-tests/printf/formatAttribute/AttributeFormattingFunction.cpp

@@ -0,0 +1,6 @@
+
+typedef void *va_list;
+
+int myPrintf(const char *format, ...) __attribute__((format(printf, 1, 2)));
+int mySprintf(char *buffer, const char *format, ...) __attribute__((format(__printf__, 2, 3)));
+int myVprintf(const char *format, va_list arg) __attribute__((format(printf, 1, 0)));

cpp/ql/test/library-tests/printf/formatAttribute/AttributeFormattingFunction.expected

@@ -0,0 +1,2 @@
+| AttributeFormattingFunction.cpp:4:5:4:12 | myPrintf | 0 | char | wchar_t | wchar_t |
+| AttributeFormattingFunction.cpp:5:5:5:13 | mySprintf | 1 | char | wchar_t | wchar_t |

cpp/ql/test/library-tests/printf/formatAttribute/AttributeFormattingFunction.ql

@@ -0,0 +1,5 @@
+import cpp
+
+from AttributeFormattingFunction f
+select f, f.getFormatParameterIndex(), concat(f.getDefaultCharType().toString(), ", "),
+  concat(f.getWideCharType().toString(), ", "), concat(f.getNonDefaultCharType().toString(), ", ")

cpp/ql/test/library-tests/printf/formatAttribute/FormatAttribute.expected

@@ -0,0 +1,3 @@
+| AttributeFormattingFunction.cpp:4:54:4:59 | format | printf | 0 | 1 |
+| AttributeFormattingFunction.cpp:5:69:5:74 | format | __printf__ | 1 | 2 |
+| AttributeFormattingFunction.cpp:6:63:6:68 | format | printf | 0 |  |

cpp/ql/test/library-tests/printf/formatAttribute/FormatAttribute.ql

@@ -0,0 +1,5 @@
+import cpp
+
+from FormatAttribute fa
+select fa, fa.getArchetype(), concat(fa.getFormatIndex().toString(), ", "),
+  concat(fa.getFirstFormatArgIndex().toString(), ", ")

cpp/ql/test/library-tests/printf/formatLiteral/formatLiteral.expected

@@ -0,0 +1,53 @@
+| test.c:14:9:14:10 |  | 1 |
+| test.c:15:9:15:14 |   | 2 |
+| test.c:16:9:16:12 | \t | 2 |
+| test.c:17:9:17:12 | %% | 2 |
+| test.c:20:9:20:12 | %c | 2 |
+| test.c:21:9:21:16 | %c%c%c | 4 |
+| test.c:24:9:24:23 | Hello, world! | 14 |
+| test.c:25:9:25:12 | %s | 14 |
+| test.c:26:9:26:14 | %.4s | 5 |
+| test.c:27:9:27:16 | %s, %s | 14 |
+| test.c:30:9:30:12 | %i | 12 |
+| test.c:31:9:31:14 | %lli | 12 |
+| test.c:32:9:32:12 | %i | 12 |
+| test.c:33:9:33:14 | %lli | 21 |
+| test.c:34:9:34:12 | %d | 12 |
+| test.c:35:9:35:12 | %u | 11 |
+| test.c:36:9:36:12 | %x | 9 |
+| test.c:37:9:37:12 | %X | 9 |
+| test.c:38:9:38:13 | %#x | 11 |
+| test.c:39:9:39:12 | %o | 12 |
+| test.c:40:9:40:13 | %#o | 13 |
+| test.c:43:9:43:12 | %f | 318 |
+| test.c:44:9:44:14 | %.2f | 314 |
+| test.c:45:9:45:12 | %e | 15 |
+| test.c:59:10:59:14 | %Ii | 12 |
+| test.c:66:10:66:14 | %zu | 21 |
+| test.c:67:10:67:14 | %Zu | 21 |
+| test.c:74:10:74:14 | %lc | 2 |
+| test.c:78:9:78:20 | %2$i, %1$i | 5 |
+| test.c:79:9:79:20 | %2$i, %1$i | 25 |
+| test.c:81:9:81:24 | %2$02i %1$4.2f |  |
+| test.c:85:10:85:18 | %2$*1$d |  |
+| test.c:86:10:86:19 | %2$0*1$d |  |
+| test.c:92:10:92:19 | %2$.*1$f |  |
+| test.c:99:10:99:12 | # | 2 |
+| test.c:100:10:100:13 | %% | 2 |
+| test.c:101:10:101:15 | %%%% | 3 |
+| test.c:102:10:102:15 | %%%f | 319 |
+| test.c:103:10:103:17 | %%%%%f | 320 |
+| test.c:104:10:104:18 | %4.2f%% | 315 |
+| test.c:105:10:105:17 | %%%f%% | 320 |
+| test.c:112:10:112:13 | %f | 318 |
+| test.c:113:10:113:15 | %.1f | 313 |
+| test.c:114:10:114:14 | %1f | 318 |
+| test.c:115:10:115:16 | %1.1f | 313 |
+| test.c:116:10:116:13 | %e | 15 |
+| test.c:117:10:117:15 | %.2e | 11 |
+| test.c:118:10:118:14 | %3e | 15 |
+| test.c:119:10:119:16 | %3.2e | 11 |
+| test.c:120:10:120:13 | %g | 15 |
+| test.c:121:10:121:15 | %.1g | 10 |
+| test.c:122:10:122:14 | %4g | 15 |
+| test.c:123:10:123:16 | %4.1g | 10 |

cpp/ql/test/library-tests/printf/formatLiteral/formatLiteral.ql

@@ -0,0 +1,4 @@
+import semmle.code.cpp.commons.Printf
+
+from FormatLiteral fl
+select fl, concat(fl.getMaxConvertedLength().toString(), ", ")

cpp/ql/test/library-tests/printf/formatLiteral/formatType.expected

@@ -0,0 +1,51 @@
+| test.c:20:9:20:12 | %c | 0 |  | c |  | file://:0:0:0:0 | char |
+| test.c:21:9:21:16 | %c%c%c | 0 |  | c |  | file://:0:0:0:0 | char |
+| test.c:21:9:21:16 | %c%c%c | 1 |  | c |  | file://:0:0:0:0 | char |
+| test.c:21:9:21:16 | %c%c%c | 2 |  | c |  | file://:0:0:0:0 | char |
+| test.c:25:9:25:12 | %s | 0 |  | s |  | file://:0:0:0:0 | char * |
+| test.c:26:9:26:14 | %.4s | 0 |  | s |  | file://:0:0:0:0 | char * |
+| test.c:27:9:27:16 | %s, %s | 0 |  | s |  | file://:0:0:0:0 | char * |
+| test.c:27:9:27:16 | %s, %s | 1 |  | s |  | file://:0:0:0:0 | char * |
+| test.c:30:9:30:12 | %i | 0 |  | i |  | file://:0:0:0:0 | int |
+| test.c:31:9:31:14 | %lli | 0 |  | i | ll | file://:0:0:0:0 | long long |
+| test.c:32:9:32:12 | %i | 0 |  | i |  | file://:0:0:0:0 | int |
+| test.c:33:9:33:14 | %lli | 0 |  | i | ll | file://:0:0:0:0 | long long |
+| test.c:34:9:34:12 | %d | 0 |  | d |  | file://:0:0:0:0 | int |
+| test.c:35:9:35:12 | %u | 0 |  | u |  | file://:0:0:0:0 | unsigned int |
+| test.c:36:9:36:12 | %x | 0 |  | x |  | file://:0:0:0:0 | unsigned int |
+| test.c:37:9:37:12 | %X | 0 |  | X |  | file://:0:0:0:0 | unsigned int |
+| test.c:38:9:38:13 | %#x | 0 |  | x |  | file://:0:0:0:0 | unsigned int |
+| test.c:39:9:39:12 | %o | 0 |  | o |  | file://:0:0:0:0 | unsigned int |
+| test.c:40:9:40:13 | %#o | 0 |  | o |  | file://:0:0:0:0 | unsigned int |
+| test.c:43:9:43:12 | %f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:44:9:44:14 | %.2f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:45:9:45:12 | %e | 0 |  | e |  | file://:0:0:0:0 | double |
+| test.c:59:10:59:14 | %Ii | 0 |  | i |  | file://:0:0:0:0 | int |
+| test.c:66:10:66:14 | %zu | 0 |  | u | z | test.c:50:27:50:32 | size_t |
+| test.c:67:10:67:14 | %Zu | 0 |  | u | Z | test.c:50:27:50:32 | size_t |
+| test.c:74:10:74:14 | %lc | 0 |  | c | l | file://:0:0:0:0 | wchar_t |
+| test.c:78:9:78:20 | %2$i, %1$i | 0 | 2$ | i |  | file://:0:0:0:0 | int |
+| test.c:78:9:78:20 | %2$i, %1$i | 1 | 1$ | i |  | file://:0:0:0:0 | int |
+| test.c:79:9:79:20 | %2$i, %1$i | 0 | 2$ | i |  | file://:0:0:0:0 | int |
+| test.c:79:9:79:20 | %2$i, %1$i | 1 | 1$ | i |  | file://:0:0:0:0 | int |
+| test.c:81:9:81:24 | %2$02i %1$4.2f | 0 | 2$ | i |  | file://:0:0:0:0 | int |
+| test.c:81:9:81:24 | %2$02i %1$4.2f | 1 | 1$ | f |  | file://:0:0:0:0 | double |
+| test.c:85:10:85:18 | %2$*1$d | 0 | 2$ | d |  | file://:0:0:0:0 | int |
+| test.c:86:10:86:19 | %2$0*1$d | 0 | 2$ | d |  | file://:0:0:0:0 | int |
+| test.c:92:10:92:19 | %2$.*1$f | 0 | 2$ | f |  | file://:0:0:0:0 | double |
+| test.c:102:10:102:15 | %%%f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:103:10:103:17 | %%%%%f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:104:10:104:18 | %4.2f%% | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:105:10:105:17 | %%%f%% | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:112:10:112:13 | %f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:113:10:113:15 | %.1f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:114:10:114:14 | %1f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:115:10:115:16 | %1.1f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:116:10:116:13 | %e | 0 |  | e |  | file://:0:0:0:0 | double |
+| test.c:117:10:117:15 | %.2e | 0 |  | e |  | file://:0:0:0:0 | double |
+| test.c:118:10:118:14 | %3e | 0 |  | e |  | file://:0:0:0:0 | double |
+| test.c:119:10:119:16 | %3.2e | 0 |  | e |  | file://:0:0:0:0 | double |
+| test.c:120:10:120:13 | %g | 0 |  | g |  | file://:0:0:0:0 | double |
+| test.c:121:10:121:15 | %.1g | 0 |  | g |  | file://:0:0:0:0 | double |
+| test.c:122:10:122:14 | %4g | 0 |  | g |  | file://:0:0:0:0 | double |
+| test.c:123:10:123:16 | %4.1g | 0 |  | g |  | file://:0:0:0:0 | double |

cpp/ql/test/library-tests/printf/formatLiteral/formatType.ql

@@ -0,0 +1,6 @@
+import semmle.code.cpp.commons.Printf
+
+from FormatLiteral fl, int i
+select fl, i, concat(fl.getParameterField(i).toString(), ", "), fl.getConversionChar(i),
+  fl.getLength(i), concat(fl.getConversionType(i).getLocation().toString(), ", "),
+  concat(fl.getConversionType(i).toString(), ", ")

cpp/ql/test/library-tests/printf/formatLiteral/test.c

@@ -0,0 +1,125 @@
+/** standard printf functions */
+
+int printf(const char *format, ...);
+
+/** test program */
+
+int main(int argc, char *argv[])
+{
+	long long int lli;
+	double d;
+	int i;
+
+	// constant expressions
+	printf("");
+	printf("\x20");
+	printf("\t");
+	printf("%%");
+
+	// characters
+	printf("%c", 'a');
+	printf("%c%c%c", 'a', 'b', 'c');
+
+	// strings
+	printf("Hello, world!");
+	printf("%s", "Hello, world!");
+	printf("%.4s", "Hello, world!");
+	printf("%s, %s", "Hello", "world!");
+
+	// integers
+	printf("%i", i);
+	printf("%lli", i);
+	printf("%i", lli);
+	printf("%lli", lli);
+	printf("%d", i);
+	printf("%u", i);
+	printf("%x", i);
+	printf("%X", i);
+	printf("%#x", i);
+	printf("%o", i);
+	printf("%#o", i);
+
+	// doubles
+	printf("%f", d);
+	printf("%.2f", d);
+	printf("%e", d);
+
+	return 0;
+}
+
+typedef long unsigned int size_t;
+typedef unsigned int wint_t;
+
+void more_cases(int a, int b)
+{
+	// integers
+	{
+		int i;
+
+		printf("%Ii", i); // glibc 2.2 'I' prefix
+	}
+
+	// size_t
+	{
+		size_t st;
+
+		printf("%zu", st); // size_t
+		printf("%Zu", st); // non-standard synonym for 'z'
+	}
+
+	// wint_t
+	{
+		wint_t wt;
+
+		printf("%lc", wt); // wide character
+	}
+
+	// posix indexed format arguments
+	printf("%2$i, %1$i", 1, 2); // '2, 1'
+	printf("%2$i, %1$i", a, b);
+	
+	printf("%2$02i %1$4.2f", 3.3333f, 6); // 06, 3.33
+	{
+		int width, num;
+
+		printf("%2$*1$d", width, num);
+		printf("%2$0*1$d", width, num);
+	}
+	{
+		int precision;
+		float num;
+
+		printf("%2$.*1$f", precision, num);
+	}
+
+	// %%
+	{
+		float num;
+
+		printf("#");
+		printf("%%");
+		printf("%%%%");
+		printf("%%%f", num);
+		printf("%%%%%f", num);
+		printf("%4.2f%%", num);
+		printf("%%%f%%", num);
+	}
+
+	// more tests of width and precision
+	{
+		float num;
+
+		printf("%f", num);
+		printf("%.1f", num);
+		printf("%1f", num);
+		printf("%1.1f", num);
+		printf("%e", num);
+		printf("%.2e", num);
+		printf("%3e", num);
+		printf("%3.2e", num);
+		printf("%g", num);
+		printf("%.1g", num);
+		printf("%4g", num);
+		printf("%4.1g", num);
+	}
+}

cpp/ql/test/library-tests/templates/CPP-223/decls.expected

@@ -13,6 +13,7 @@
 | test.cpp:4:26:4:26 | c<<expression>> |
 | test.cpp:4:26:4:26 | c<<unnamed>> |
 | test.cpp:5:29:5:29 | e |
+| test.cpp:6:24:6:24 | f |
 | test.cpp:6:26:6:26 | (unnamed parameter 0) |
 | test.cpp:6:29:6:31 | (unnamed parameter 1) |
 | test.cpp:7:20:7:20 | f |

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.3.2
+version: 1.3.3-dev
 groups:
     - csharp
     - solorigate

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.3.2
+version: 1.3.3-dev
 groups:
     - csharp
     - solorigate

csharp/ql/integration-tests/qlpack.yml

@@ -1,2 +1,2 @@
-libraryPathDependencies:
-  - codeql-csharp
+dependencies:
+  codeql/csharp-all: '*'

csharp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-all
-version: 0.4.2
+version: 0.4.3-dev
 groups: csharp
 dbscheme: semmlecode.csharp.dbscheme
 extractor: csharp

csharp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-queries
-version: 0.4.2
+version: 0.4.3-dev
 groups: 
   - csharp
   - queries

csharp/ql/test/TestUtilities/InlineExpectationsTest.qll

@@ -137,6 +137,7 @@ abstract class InlineExpectationsTest extends string {
   final predicate hasFailureMessage(FailureLocatable element, string message) {
     exists(ActualResult actualResult |
       actualResult.getTest() = this and
+      actualResult.getTag() = this.getARelevantTag() and
       element = actualResult and
       (
         exists(FalseNegativeExpectation falseNegative |
@@ -150,9 +151,18 @@ abstract class InlineExpectationsTest extends string {
       )
     )
     or
+    exists(ActualResult actualResult |
+      actualResult.getTest() = this and
+      not actualResult.getTag() = this.getARelevantTag() and
+      element = actualResult and
+      message =
+        "Tag mismatch: Actual result with tag '" + actualResult.getTag() +
+          "' that is not part of getARelevantTag()"
+    )
+    or
     exists(ValidExpectation expectation |
       not exists(ActualResult actualResult | expectation.matchesActualResult(actualResult)) and
-      expectation.getTag() = getARelevantTag() and
+      expectation.getTag() = this.getARelevantTag() and
       element = expectation and
       (
         expectation instanceof GoodExpectation and

docs/change-notes.md

@@ -45,7 +45,7 @@ The valid YAML properties in the metadata are:
 After the `---` line following the metadata, the rest of the markdown file is the user-visible content of the change note. This should usually be a single markdown bullet list entry (starting with `*`), although it is acceptable to have multiple bullet entries in the same change note if there are multiple changes that are closely related and have the same category metadata.
 
 ## Change categories
-Each change note must specifiy a `category` property in its metadata. This category servers two purposes: It determines how the change affects the version number of the next release of the pack, and it is used to group related changes in the final changelog. There is one set of available categories for query packs, and another set of available categories for library packs.
+Each change note must specify a `category` property in its metadata. This category servers two purposes: It determines how the change affects the version number of the next release of the pack, and it is used to group related changes in the final changelog. There is one set of available categories for query packs, and another set of available categories for library packs.
 
 ### Query pack change categories
 | Category       | SemVer effect      | Description |

docs/codeql/codeql-for-visual-studio-code/exploring-data-flow-with-path-queries.rst

@@ -28,7 +28,7 @@ Running path queries in VS Code
 #. Once the query has finished running, you can see the results in the Results view as usual (under ``alerts`` in the dropdown menu). Each query result describes the flow of information between a source and a sink.
 #. Expand the result to see the individual steps that the data follows. 
 #. Click each step to jump to it in the source code and investigate the problem further.
-#. To navigate the path from your keyboard, you can bind shortcuts to the **CodeQL: Show Previous Step on Path** and **CodeQL: Show Next Step on Path** commands.
+#. To navigate the results from your keyboard, you can bind shortcuts to the **CodeQL: Navigate Up/Down/Left/Right in Result Viewer** commands.
 
 Further reading
 -----------------

docs/codeql/ql-language-reference/modules.rst

@@ -273,4 +273,71 @@ The ``<module_expression>`` itself can be a module name, a selection, or a quali
 reference. For more information, see ":ref:`name-resolution`."
 
 For information about how import statements are looked up, see "`Module resolution <https://codeql.github.com/docs/ql-language-reference/ql-language-specification/#module-resolution>`__"
-in the QL language specification. 
+in the QL language specification. 
+
+Built-in modules
+****************
+
+QL defines a ``QlBuiltins`` module that is always in scope.
+Currently, it defines a single parameterized sub-module
+``EquivalenceRelation``, that provides an efficient abstraction for working with
+(partial) equivalence relations in QL.
+
+Equivalence relations
+=====================
+
+The built-in ``EquivalenceRelation`` module is parameterized by a type ``T`` and a
+binary base relation ``base`` on ``T``. The symmetric and transitive closure of ``base``
+induces a partial equivalence relation on ``T``. If every value of ``T`` appears in
+``base``, then the induced relation is an equivalence relation on ``T``.
+
+The ``EquivalenceRelation`` module exports a ``getEquivalenceClass`` predicate that
+gets the equivalence class, if any, associated with a given ``T`` element by the
+(partial) equivalence relation induced by ``base``.
+
+The following example illustrates an application of the ``EquivalenceRelation``
+module to generate a custom equivalence relation:
+
+.. code-block:: ql
+
+  class Node extends int {
+    Node() { this in [1 .. 6] }
+  }
+
+  predicate base(Node x, Node y) {
+    x = 1 and y = 2
+    or
+    x = 3 and y = 4
+  }
+
+  module Equiv = QlBuiltins::EquivalenceRelation<Node, base/2>;
+
+  from int x, int y
+  where Equiv::getEquivalenceClass(x) = Equiv::getEquivalenceClass(y)
+  select x, y
+
+Since ``base`` does not relate ``5`` or ``6`` to any nodes, the induced
+relation is a partial equivalence relation on ``Node`` and does not relate ``5``
+or ``6`` to any nodes either.
+
+The above select clause returns the following partial equivalence relation:
+
++---+---+
+| x | y |
++===+===+
+| 1 | 1 |
++---+---+
+| 1 | 2 |
++---+---+
+| 2 | 1 |
++---+---+
+| 2 | 2 |
++---+---+
+| 3 | 3 |
++---+---+
+| 3 | 4 |
++---+---+
+| 4 | 3 |
++---+---+
+| 4 | 4 |
++---+---+

docs/codeql/ql-language-reference/ql-language-specification.rst

@@ -639,7 +639,7 @@ Various kinds of syntax can have *annotations* applied to them. Annotations are
                     |   "override"
                     |   "query"
 
-   argsAnnotation ::= "pragma" "[" ("inline" | "noinline" | "nomagic" | "noopt") "]"
+   argsAnnotation ::= "pragma" "[" ("inline" | "noinline" | "nomagic" | "noopt" | "assume_small_delta") "]"
                   |   "language" "[" "monotonicAggregates" "]"
                   |   "bindingset" "[" (variable ( "," variable)*)? "]"
 
@@ -687,17 +687,19 @@ Parameterized annotations take some additional arguments.
 
 The parameterized annotation ``pragma`` supplies compiler pragmas, and may be applied in various contexts depending on the pragma in question.
 
-+--------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
-| Pragma       | Classes | Characters | Member predicates | Non-member predicates | Imports | Fields | Modules | Aliases |
-+==============+=========+============+===================+=======================+=========+========+=========+=========+
-| ``inline``   |         | yes        | yes               | yes                   |         |        |         |         |
-+--------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
-| ``noinline`` |         | yes        | yes               | yes                   |         |        |         |         |
-+--------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
-| ``nomagic``  |         | yes        | yes               | yes                   |         |        |         |         |
-+--------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
-| ``noopt``    |         | yes        | yes               | yes                   |         |        |         |         |
-+--------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
++---------------------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
+| Pragma                    | Classes | Characters | Member predicates | Non-member predicates | Imports | Fields | Modules | Aliases |
++===========================+=========+============+===================+=======================+=========+========+=========+=========+
+| ``inline``                |         | yes        | yes               | yes                   |         |        |         |         |
++---------------------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
+| ``noinline``              |         | yes        | yes               | yes                   |         |        |         |         |
++---------------------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
+| ``nomagic``               |         | yes        | yes               | yes                   |         |        |         |         |
++---------------------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
+| ``noopt``                 |         | yes        | yes               | yes                   |         |        |         |         |
++---------------------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
+| ``assume_small_delta``    |         | yes        | yes               | yes                   |         |        |         |         |
++---------------------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
 
 The parameterized annotation ``language`` supplies language pragmas which change the behavior of the language. Language pragmas apply at the scope level, and are inherited by nested scopes.
 
@@ -2048,7 +2050,7 @@ The complete grammar for QL is as follows:
                     |   "override"
                     |   "query"
 
-   argsAnnotation ::= "pragma" "[" ("noinline" | "nomagic" | "noopt") "]"
+   argsAnnotation ::= "pragma" "[" ("inline" | "noinline" | "nomagic" | "noopt" | "assume_small_delta") "]"
                   |   "language" "[" "monotonicAggregates" "]"
                   |   "bindingset" "[" (variable ( "," variable)*)? "]"
 

docs/codeql/ql-language-reference/signatures.rst

@@ -40,10 +40,11 @@ Type signatures
 ===============
 
 Type signatures declare module parameters that will be substituted with types when the module is instantiated.
-Type signatures are used to specify supertypes and are the simplest category of signatures.
+Type signatures may specify supertypes and required member predicates (in addition to those member predicates that are
+implied by the supertypes).
 
 The substitution of type signatures relies on structural typing. That is, types do not have to be explicitly defined as
-implementing a type signature - they just need to have the specified (transitive) supertypes.
+implementing a type signature - they just need to have the specified (transitive) supertypes and member predicates.
 
 In detail, a type signature definition consists of:
 
@@ -52,7 +53,8 @@ In detail, a type signature definition consists of:
 #. The name of the type signature. This is an `identifier <https://codeql.github.com/docs/ql-language-reference/ql-language-specification/#identifiers>`_
    starting with a uppercase letter.
 #. Optionally, the keyword ``extends`` followed by a list of types, separated by commas.
-#. A semicolon ``;``.
+#. Either a semicolon ``;`` or a list of predicate signatures enclosed in braces.
+   The ``signature`` keyword is omitted for these contained signatures.
 
 For example:
 
@@ -60,6 +62,10 @@ For example:
 
     signature class ExtendsInt extends int;
 
+    signature class CanBePrinted {
+      string toString();
+    }
+
 Module signatures
 =================
 

docs/ql-libraries/dataflow/dataflow.md

@@ -294,8 +294,8 @@ through an additional step targeting a `PostUpdateNode`).
 
 It is recommended to introduce `PostUpdateNode`s for all `ArgumentNode`s (this
 can be skipped for immutable arguments), and all field qualifiers for both
-reads and stores. Note also that in the case of compund arguments, such as
-`b ? x : y`, it is recommented to have post-update nodes for `x` and `y` (and
+reads and stores. Note also that in the case of compound arguments, such as
+`b ? x : y`, it is recommended to have post-update nodes for `x` and `y` (and
 not the compound argument itself), and let `[post update] x` have both `x`
 and `b ? x : y` as pre-update nodes (and similarly for `[post update] y`).
 

go/old-change-notes/2020-11-12-zipslip-sanitizers.md

@@ -1,2 +1,2 @@
 lgtm,codescanning
-* Improved recongition of sanitizer functions for the `go/zipslip` query. This may reduce false-positives (but also perhaps false-negatives) when application code attempts to check a zip header entry does not contain an illegal path traversal attempt.
+* Improved recognition of sanitizer functions for the `go/zipslip` query. This may reduce false-positives (but also perhaps false-negatives) when application code attempts to check a zip header entry does not contain an illegal path traversal attempt.

go/old-change-notes/2021-01-12-model-couchbase.md

@@ -1,2 +1,2 @@
 lgtm,codescanning
-* Added support for [the offical Couchbase Go SDK library](https://github.com/couchbase/gocb), v1 and v2. The `go/sql-injection` query (which also handles non-SQL databases such as Couchbase) will now identify Couchbase queries built from untrusted external input.
+* Added support for [the official Couchbase Go SDK library](https://github.com/couchbase/gocb), v1 and v2. The `go/sql-injection` query (which also handles non-SQL databases such as Couchbase) will now identify Couchbase queries built from untrusted external input.

go/ql/config/legacy-support/qlpack.yml

@@ -1,3 +1,4 @@
 name: legacy-libraries-go
 version: 0.0.0
+# Note libraryPathDependencies is obsolete and should not be used in new qlpacks.
 libraryPathDependencies: codeql-go

go/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-all
-version: 0.3.2
+version: 0.3.3-dev
 groups: go
 dbscheme: go.dbscheme
 extractor: go

go/ql/src/Diagnostics/SuccessfullyExtractedFiles.ql

@@ -9,5 +9,7 @@
 import go
 
 from File f
-where not exists(Error e | e.getFile() = f)
-select f.getRelativePath()
+where
+  not exists(Error e | e.getFile() = f) and
+  exists(f.getRelativePath())
+select f, ""

go/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-queries
-version: 0.3.2
+version: 0.3.3-dev
 groups:
   - go
   - queries

go/ql/test/TestUtilities/InlineExpectationsTest.qll

@@ -137,6 +137,7 @@ abstract class InlineExpectationsTest extends string {
   final predicate hasFailureMessage(FailureLocatable element, string message) {
     exists(ActualResult actualResult |
       actualResult.getTest() = this and
+      actualResult.getTag() = this.getARelevantTag() and
       element = actualResult and
       (
         exists(FalseNegativeExpectation falseNegative |
@@ -150,9 +151,18 @@ abstract class InlineExpectationsTest extends string {
       )
     )
     or
+    exists(ActualResult actualResult |
+      actualResult.getTest() = this and
+      not actualResult.getTag() = this.getARelevantTag() and
+      element = actualResult and
+      message =
+        "Tag mismatch: Actual result with tag '" + actualResult.getTag() +
+          "' that is not part of getARelevantTag()"
+    )
+    or
     exists(ValidExpectation expectation |
       not exists(ActualResult actualResult | expectation.matchesActualResult(actualResult)) and
-      expectation.getTag() = getARelevantTag() and
+      expectation.getTag() = this.getARelevantTag() and
       element = expectation and
       (
         expectation instanceof GoodExpectation and

go/ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.expected

@@ -1 +1 @@
-| query-tests/Diagnostics/util.go |
+| util.go:0:0:0:0 | util.go |  |

java/documentation/library-coverage/coverage.rst

@@ -8,16 +8,20 @@ Java framework & library support
 
    Framework / library,Package,Flow sources,Taint & value steps,Sinks (total),`CWE‑022` :sub:`Path injection`,`CWE‑036` :sub:`Path traversal`,`CWE‑079` :sub:`Cross-site scripting`,`CWE‑089` :sub:`SQL injection`,`CWE‑090` :sub:`LDAP injection`,`CWE‑094` :sub:`Code injection`,`CWE‑319` :sub:`Cleartext transmission`
    Android,``android.*``,52,479,116,,,3,67,,,
+   Android extensions,``androidx.*``,5,183,8,,,,,,,
    `Apache Commons Collections <https://commons.apache.org/proper/commons-collections/>`_,"``org.apache.commons.collections``, ``org.apache.commons.collections4``",,1600,,,,,,,,
    `Apache Commons IO <https://commons.apache.org/proper/commons-io/>`_,``org.apache.commons.io``,,556,106,91,,,,,,15
    `Apache Commons Lang <https://commons.apache.org/proper/commons-lang/>`_,``org.apache.commons.lang3``,,424,,,,,,,,
    `Apache Commons Text <https://commons.apache.org/proper/commons-text/>`_,``org.apache.commons.text``,,272,,,,,,,,
    `Apache HttpComponents <https://hc.apache.org/>`_,"``org.apache.hc.core5.*``, ``org.apache.http``",5,136,28,,,3,,,,25
+   `Apache Log4j 2 <https://logging.apache.org/log4j/2.0/>`_,``org.apache.logging.log4j``,,8,359,,,,,,,
    `Google Guava <https://guava.dev/>`_,``com.google.common.*``,,728,39,,6,,,,,
+   JBoss Logging,``org.jboss.logging``,,,324,,,,,,,
    `JSON-java <https://github.com/stleary/JSON-java>`_,``org.json``,,236,,,,,,,,
    Java Standard Library,``java.*``,3,589,130,28,,,7,,,10
    Java extensions,"``javax.*``, ``jakarta.*``",63,609,32,,,4,,1,1,2
+   Kotlin Standard Library,``kotlin*``,,1835,12,10,,,,,,2
    `Spring <https://spring.io/>`_,``org.springframework.*``,29,477,101,,,,19,14,,29
-   Others,"``androidx.core.app``, ``androidx.slice``, ``cn.hutool.core.codec``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.core``, ``com.fasterxml.jackson.databind``, ``com.hubspot.jinjava``, ``com.mitchellbosecke.pebble``, ``com.opensymphony.xwork2.ognl``, ``com.rabbitmq.client``, ``com.unboundid.ldap.sdk``, ``com.zaxxer.hikari``, ``flexjson``, ``freemarker.cache``, ``freemarker.template``, ``groovy.lang``, ``groovy.util``, ``jodd.json``, ``kotlin``, ``net.sf.saxon.s9api``, ``ognl``, ``okhttp3``, ``org.apache.commons.codec``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.logging``, ``org.apache.commons.ognl``, ``org.apache.directory.ldap.client.api``, ``org.apache.ibatis.jdbc``, ``org.apache.log4j``, ``org.apache.logging.log4j``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.apache.velocity.app``, ``org.apache.velocity.runtime``, ``org.codehaus.groovy.control``, ``org.dom4j``, ``org.hibernate``, ``org.jboss.logging``, ``org.jdbi.v3.core``, ``org.jooq``, ``org.mvel2``, ``org.scijava.log``, ``org.slf4j``, ``org.thymeleaf``, ``org.xml.sax``, ``org.xmlpull.v1``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``, ``retrofit2``",65,2326,972,10,,,14,18,,5
+   Others,"``cn.hutool.core.codec``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.core``, ``com.fasterxml.jackson.databind``, ``com.hubspot.jinjava``, ``com.mitchellbosecke.pebble``, ``com.opensymphony.xwork2.ognl``, ``com.rabbitmq.client``, ``com.unboundid.ldap.sdk``, ``com.zaxxer.hikari``, ``flexjson``, ``freemarker.cache``, ``freemarker.template``, ``groovy.lang``, ``groovy.util``, ``jodd.json``, ``net.sf.saxon.s9api``, ``ognl``, ``okhttp3``, ``org.apache.commons.codec``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.logging``, ``org.apache.commons.ognl``, ``org.apache.directory.ldap.client.api``, ``org.apache.ibatis.jdbc``, ``org.apache.log4j``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.apache.velocity.app``, ``org.apache.velocity.runtime``, ``org.codehaus.groovy.control``, ``org.dom4j``, ``org.hibernate``, ``org.jdbi.v3.core``, ``org.jooq``, ``org.mvel2``, ``org.scijava.log``, ``org.slf4j``, ``org.thymeleaf``, ``org.xml.sax``, ``org.xmlpull.v1``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``, ``retrofit2``",60,300,269,,,,14,18,,3
    Totals,,217,8432,1524,129,6,10,107,33,1,86
 

java/documentation/library-coverage/frameworks.csv

@@ -1,12 +1,16 @@
 Framework name,URL,Package prefixes
 Java Standard Library,,java.*
 Java extensions,,javax.* jakarta.*
+Kotlin Standard Library,,kotlin*
+Android,,android.*
+Android extensions,,androidx.*
 Apache Commons Collections,https://commons.apache.org/proper/commons-collections/,org.apache.commons.collections org.apache.commons.collections4
 Apache Commons IO,https://commons.apache.org/proper/commons-io/,org.apache.commons.io
 Apache Commons Lang,https://commons.apache.org/proper/commons-lang/,org.apache.commons.lang3
 Apache Commons Text,https://commons.apache.org/proper/commons-text/,org.apache.commons.text
 Apache HttpComponents,https://hc.apache.org/,org.apache.hc.core5.* org.apache.http
-Android,,android.*
+Apache Log4j 2,https://logging.apache.org/log4j/2.0/,org.apache.logging.log4j
 Google Guava,https://guava.dev/,com.google.common.*
+JBoss Logging,,org.jboss.logging
 JSON-java,https://github.com/stleary/JSON-java,org.json
 Spring,https://spring.io/,org.springframework.*

java/kotlin-extractor/kotlin_plugin_versions.py

@@ -49,7 +49,7 @@ def get_single_version(fakeVersionOutput = None):
     matching_minor_versions.sort(reverse = True)
 
     for version in matching_minor_versions:
-        if version <= current_version:
+        if version[0:3] <= current_version[0:3]:
             return version_tuple_to_string(version)
 
     return version_tuple_to_string(matching_minor_versions[-1])

java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java

@@ -277,7 +277,7 @@ public class OdasaOutput {
 				// Only re-write an existing trap file if we encountered a newer version of the same class.
 				TrapClassVersion trapVersion = readVersionInfo(trap);
 				if (!currVersion.isValid()) {
-					log.warn("Not rewriting trap file for: " + shortName + " " + trapVersion + " " + currVersion + " " + trap);
+					log.trace("Not rewriting trap file for: " + shortName + " " + trapVersion + " " + currVersion + " " + trap);
 				} else if (currVersion.newerThan(trapVersion)) {
 					log.trace("Rewriting trap file for: " + shortName + " " + trapVersion + " " + currVersion + " " + trap);
 					deleteTrapFileAndDependencies(sym, signature);
@@ -291,7 +291,7 @@ public class OdasaOutput {
 			// If the TRAP file already exists then we
 			// don't need to write it.
 			if (trap.exists()) {
-				log.warn("Not rewriting trap file for " + trap.toString() + " as it exists");
+				log.trace("Not rewriting trap file for " + trap.toString() + " as it exists");
 				return null;
 			}
 			// If the TRAP file was written in the past, and
@@ -301,7 +301,7 @@ public class OdasaOutput {
 			File trapFileDir = trap.getParentFile();
 			File trapOld = new File(trapFileDir, trap.getName().replace(".trap.gz", ".trap-old.gz"));
 			if (trapOld.exists()) {
-				log.warn("Not rewriting trap file for " + trap.toString() + " as the trap-old exists");
+				log.trace("Not rewriting trap file for " + trap.toString() + " as the trap-old exists");
 				return null;
 			}
 			// Otherwise, if any newer TRAP file has already
@@ -316,7 +316,7 @@ public class OdasaOutput {
 					if (m.matches() && m.group(1).equals(trapFileBaseName)) {
 						TrapClassVersion v = new TrapClassVersion(Integer.valueOf(m.group(2)), Integer.valueOf(m.group(3)), Long.valueOf(m.group(4)), m.group(5));
 						if (v.newerThan(trapFileVersion)) {
-							log.warn("Not rewriting trap file for " + trap.toString() + " as " + f.toString() + " exists");
+							log.trace("Not rewriting trap file for " + trap.toString() + " as " + f.toString() + " exists");
 							return null;
 						}
 					}

java/kotlin-extractor/src/main/java/com/semmle/util/trap/pathtransformers/PathTransformer.java

@@ -14,7 +14,15 @@ public abstract class PathTransformer {
 	 * canonical, absolute, strings and normalises away Unix/Windows differences.
 	 */
 	public String fileAsDatabaseString(File file) {
-		String path;
+		String path = file.getPath();
+		// For /!unknown-binary-location/... and /modules/...
+		// paths, on Windows the standard code wants to
+		// normalise them to e.g. C:/!unknown-binary-location/...
+		// which is particularly annoying for cross-platform test
+		// output. We therefore handle them specially here.
+		if (path.matches("^[/\\\\](!unknown-binary-location|modules)[/\\\\].*")) {
+			return path.replace('\\', '/');
+		}
 		if (Boolean.valueOf(Env.systemEnv().get(Var.SEMMLE_PRESERVE_SYMLINKS)))
 			path = FileUtil.simplifyPath(file);
 		else
@@ -43,4 +51,4 @@ public abstract class PathTransformer {
 	public static PathTransformer std() {
 		return DEFAULT_TRANSFORMER;
 	}
-}
+}

java/kotlin-extractor/src/main/kotlin/ExternalDeclExtractor.kt

@@ -35,8 +35,6 @@ class ExternalDeclExtractor(val logger: FileLogger, val invocationTrapFile: Stri
         if (ret) externalDeclWorkList.add(Pair(d, signature))
         return ret
     }
-    fun extractLater(p: IrProperty) = extractLater(p, propertySignature)
-    fun extractLater(f: IrField) = extractLater(f, fieldSignature)
     fun extractLater(c: IrClass) = extractLater(c, "")
 
     fun extractExternalClasses() {

java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt

@@ -2,6 +2,7 @@ package com.github.codeql
 
 import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
 import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
+import org.jetbrains.kotlin.config.KotlinCompilerVersion
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.util.*
 import org.jetbrains.kotlin.ir.IrElement
@@ -139,6 +140,8 @@ class KotlinExtractorExtension(
             logger.flush()
             logger.info("Extraction for invocation TRAP file $invocationTrapFile")
             logger.flush()
+            logger.info("Kotlin version ${KotlinCompilerVersion.getVersion()}")
+            logger.flush()
             logPeakMemoryUsage(logger, "before extractor")
             if (System.getenv("CODEQL_EXTRACTOR_JAVA_KOTLIN_DUMP") == "true") {
                 logger.info("moduleFragment:\n" + moduleFragment.dump())

java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt

@@ -27,6 +27,7 @@ import org.jetbrains.kotlin.load.java.structure.JavaClass
 import org.jetbrains.kotlin.load.java.structure.JavaMethod
 import org.jetbrains.kotlin.load.java.structure.JavaTypeParameter
 import org.jetbrains.kotlin.load.java.structure.JavaTypeParameterListOwner
+import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.types.Variance
 import org.jetbrains.kotlin.util.OperatorNameConventions
@@ -98,15 +99,29 @@ open class KotlinFileExtractor(
         }
     }
 
+    private fun javaBinaryDeclaresMethod(c: IrClass, name: String) =
+        ((c.source as? JavaSourceElement)?.javaElement as? BinaryJavaClass)?.methods?.any { it.name.asString() == name }
+
+    private fun isJavaBinaryDeclaration(f: IrFunction) =
+        f.parentClassOrNull?.let { javaBinaryDeclaresMethod(it, f.name.asString()) } ?: false
+
+    private fun isJavaBinaryObjectMethodRedeclaration(d: IrDeclaration) =
+        when (d) {
+            is IrFunction ->
+                when (d.name.asString()) {
+                    "toString" -> d.valueParameters.isEmpty()
+                    "hashCode" -> d.valueParameters.isEmpty()
+                    "equals" -> d.valueParameters.singleOrNull()?.type?.isNullableAny() ?: false
+                    else -> false
+                } && isJavaBinaryDeclaration(d)
+            else -> false
+        }
+
     @OptIn(ObsoleteDescriptorBasedAPI::class)
     private fun isFake(d: IrDeclarationWithVisibility): Boolean {
-        val visibility = d.visibility
-        if (visibility is DelegatedDescriptorVisibility && visibility.delegate == Visibilities.InvisibleFake) {
+        val hasFakeVisibility = d.visibility.let { it is DelegatedDescriptorVisibility && it.delegate == Visibilities.InvisibleFake } || d.isFakeOverride
+        if (hasFakeVisibility && !isJavaBinaryObjectMethodRedeclaration(d))
             return true
-        }
-        if (d.isFakeOverride) {
-            return true
-        }
         try {
             if ((d as? IrFunction)?.descriptor?.isHiddenToOvercomeSignatureClash == true) {
                 return true
@@ -305,7 +320,7 @@ open class KotlinFileExtractor(
                 val kind = c.kind
                 if (kind == ClassKind.ENUM_CLASS) {
                     tw.writeIsEnumType(classId)
-                } else if (kind != ClassKind.CLASS && kind != ClassKind.OBJECT) {
+                } else if (kind != ClassKind.CLASS && kind != ClassKind.OBJECT && kind != ClassKind.ENUM_ENTRY) {
                     logger.errorElement("Unrecognised class kind $kind", c)
                 }
             }
@@ -452,7 +467,7 @@ open class KotlinFileExtractor(
                     val kind = c.kind
                     if (kind == ClassKind.ENUM_CLASS) {
                         tw.writeIsEnumType(classId)
-                    } else if (kind != ClassKind.CLASS && kind != ClassKind.OBJECT) {
+                    } else if (kind != ClassKind.CLASS && kind != ClassKind.OBJECT && kind != ClassKind.ENUM_ENTRY) {
                         logger.warnElement("Unrecognised class kind $kind", c)
                     }
 
@@ -908,7 +923,9 @@ open class KotlinFileExtractor(
             else
                 null
         } else {
-            forceExtractFunction(f, parentId, extractBody, extractMethodAndParameterTypeAccesses, typeSubstitution, classTypeArgsIncludingOuterClasses).also {
+            // Work around an apparent bug causing redeclarations of `fun toString(): String` specifically in interfaces loaded from Java classes show up like fake overrides.
+            val overriddenVisibility = if (f.isFakeOverride && isJavaBinaryObjectMethodRedeclaration(f)) OverriddenFunctionAttributes(visibility = DescriptorVisibilities.PUBLIC) else null
+            forceExtractFunction(f, parentId, extractBody, extractMethodAndParameterTypeAccesses, typeSubstitution, classTypeArgsIncludingOuterClasses, overriddenAttributes = overriddenVisibility).also {
                 // The defaults-forwarder function is a static utility, not a member, so we only need to extract this for the unspecialised instance of this class.
                 if (classTypeArgsIncludingOuterClasses.isNullOrEmpty())
                     extractDefaultsFunction(f, parentId, extractBody, extractMethodAndParameterTypeAccesses)
@@ -1884,7 +1901,7 @@ open class KotlinFileExtractor(
             IrConstImpl.defaultValueForType(0, 0, getDefaultsMethodLastArgType(callTarget))
         )
 
-        extractCallValueArguments(id, valueArgsWithDummies + extraArgs, enclosingStmt, enclosingCallable, nextIdx)
+        extractCallValueArguments(id, valueArgsWithDummies + extraArgs, enclosingStmt, enclosingCallable, nextIdx, extractVarargAsArray = true)
     }
 
     private fun getFunctionInvokeMethod(typeArgs: List<IrTypeArgument>): IrFunction? {
@@ -1961,8 +1978,12 @@ open class KotlinFileExtractor(
         superQualifierSymbol: IrClassSymbol? = null) {
 
         val locId = tw.getLocation(locElement)
+        val varargParam = syntacticCallTarget.valueParameters.withIndex().find { it.value.isVararg }
+        // If the vararg param is the only one not specified, and it has no default value, then we don't need to call a $default method,
+        // as omitting it already implies passing an empty vararg array.
+        val nullAllowedIdx = if (varargParam != null && varargParam.value.defaultValue == null) varargParam.index else -1
 
-        if (valueArguments.any { it == null }) {
+        if (valueArguments.withIndex().any { (index, it) -> it == null && index != nullAllowedIdx }) {
             extractsDefaultsCall(
                 syntacticCallTarget,
                 locId,
@@ -2082,11 +2103,11 @@ open class KotlinFileExtractor(
     private fun extractCallValueArguments(callId: Label<out DbExprparent>, call: IrFunctionAccessExpression, enclosingStmt: Label<out DbStmt>, enclosingCallable: Label<out DbCallable>, idxOffset: Int) =
         extractCallValueArguments(callId, (0 until call.valueArgumentsCount).map { call.getValueArgument(it) }, enclosingStmt, enclosingCallable, idxOffset)
 
-    private fun extractCallValueArguments(callId: Label<out DbExprparent>, valueArguments: List<IrExpression?>, enclosingStmt: Label<out DbStmt>, enclosingCallable: Label<out DbCallable>, idxOffset: Int) {
+    private fun extractCallValueArguments(callId: Label<out DbExprparent>, valueArguments: List<IrExpression?>, enclosingStmt: Label<out DbStmt>, enclosingCallable: Label<out DbCallable>, idxOffset: Int, extractVarargAsArray: Boolean = false) {
         var i = 0
         valueArguments.forEach { arg ->
             if(arg != null) {
-                if (arg is IrVararg) {
+                if (arg is IrVararg && !extractVarargAsArray) {
                     arg.elements.forEachIndexed { varargNo, vararg -> extractVarargElement(vararg, enclosingCallable, callId, i + idxOffset + varargNo, enclosingStmt) }
                     i += arg.elements.size
                 } else {

java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt

@@ -2,6 +2,7 @@ package com.github.codeql
 
 import com.github.codeql.utils.*
 import com.github.codeql.utils.versions.codeQlWithHasQuestionMark
+import com.github.codeql.utils.versions.getKotlinType
 import com.github.codeql.utils.versions.isRawType
 import com.semmle.extractor.java.OdasaOutput
 import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
@@ -22,6 +23,7 @@ import org.jetbrains.kotlin.load.java.BuiltinMethodsWithSpecialGenericSignature
 import org.jetbrains.kotlin.load.java.JvmAbi
 import org.jetbrains.kotlin.load.java.sources.JavaSourceElement
 import org.jetbrains.kotlin.load.java.structure.*
+import org.jetbrains.kotlin.load.java.typeEnhancement.hasEnhancedNullability
 import org.jetbrains.kotlin.load.kotlin.getJvmModuleNameForDeserializedDescriptor
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.NameUtils
@@ -253,19 +255,24 @@ open class KotlinUsesExtractor(
         }
     }
 
+    private fun propertySignature(p: IrProperty) =
+        ((p.getter ?: p.setter)?.extensionReceiverParameter?.let { useType(erase(it.type)).javaResult.signature } ?: "")
+
     private fun extractPropertyLaterIfExternalFileMember(p: IrProperty) {
         if (isExternalFileClassMember(p)) {
             extractExternalClassLater(p.parentAsClass)
-            dependencyCollector?.addDependency(p, externalClassExtractor.propertySignature)
-            externalClassExtractor.extractLater(p)
+            val signature = propertySignature(p) + externalClassExtractor.propertySignature
+            dependencyCollector?.addDependency(p, signature)
+            externalClassExtractor.extractLater(p, signature)
         }
     }
 
     private fun extractFieldLaterIfExternalFileMember(f: IrField) {
         if (isExternalFileClassMember(f)) {
             extractExternalClassLater(f.parentAsClass)
-            dependencyCollector?.addDependency(f, externalClassExtractor.fieldSignature)
-            externalClassExtractor.extractLater(f)
+            val signature = (f.correspondingPropertySymbol?.let { propertySignature(it.owner) } ?: "") + externalClassExtractor.fieldSignature
+            dependencyCollector?.addDependency(f, signature)
+            externalClassExtractor.extractLater(f, signature)
         }
     }
 
@@ -669,7 +676,8 @@ open class KotlinUsesExtractor(
                           otherIsPrimitive: Boolean,
                           javaClass: IrClass,
                           kotlinPackageName: String, kotlinClassName: String): TypeResults {
-            val javaResult = if ((context == TypeContext.RETURN || (context == TypeContext.OTHER && otherIsPrimitive)) && !s.isNullable() && primitiveName != null) {
+            // Note the use of `hasEnhancedNullability` here covers cases like `@NotNull Integer`, which must be extracted as `Integer` not `int`.
+            val javaResult = if ((context == TypeContext.RETURN || (context == TypeContext.OTHER && otherIsPrimitive)) && !s.isNullable() && getKotlinType(s)?.hasEnhancedNullability() != true && primitiveName != null) {
                     val label: Label<DbPrimitive> = tw.getLabelFor("@\"type;$primitiveName\"", {
                         tw.writePrimitives(it, primitiveName)
                     })
@@ -813,7 +821,7 @@ open class KotlinUsesExtractor(
                 OperatorNameConventions.INVOKE.asString())
 
         fun getSuffixIfInternal() =
-            if (f.visibility == DescriptorVisibilities.INTERNAL) {
+            if (f.visibility == DescriptorVisibilities.INTERNAL && f !is IrConstructor) {
                 "\$" + getJvmModuleName(f)
             } else {
                 ""
@@ -952,11 +960,13 @@ open class KotlinUsesExtractor(
             ((t as? IrSimpleType)?.classOrNull?.owner?.isFinalClass) != true
         }
 
-    private fun wildcardAdditionAllowed(v: Variance, t: IrType, addByDefault: Boolean) =
+    private fun wildcardAdditionAllowed(v: Variance, t: IrType, addByDefault: Boolean, javaVariance: Variance?) =
         when {
             t.hasAnnotation(jvmWildcardAnnotation) -> true
             !addByDefault -> false
             t.hasAnnotation(jvmWildcardSuppressionAnnotation) -> false
+            // If a Java declaration specifies a variance, introduce it even if it's pointless (e.g. ? extends FinalClass, or ? super Object)
+            javaVariance == v -> true
             v == Variance.IN_VARIANCE -> !(t.isNullableAny() || t.isAny())
             v == Variance.OUT_VARIANCE -> extendsAdditionAllowed(t)
             else -> false
@@ -965,14 +975,21 @@ open class KotlinUsesExtractor(
     private fun addJavaLoweringArgumentWildcards(p: IrTypeParameter, t: IrTypeArgument, addByDefault: Boolean, javaType: JavaType?): IrTypeArgument =
         (t as? IrTypeProjection)?.let {
             val newBase = addJavaLoweringWildcards(it.type, addByDefault, javaType)
+            // Note javaVariance == null means we don't have a Java type to conform to -- for example if this is a Kotlin source definition.
+            val javaVariance = javaType?.let { jType ->
+                when (jType) {
+                    is JavaWildcardType -> if (jType.isExtends) Variance.OUT_VARIANCE else Variance.IN_VARIANCE
+                    else -> Variance.INVARIANT
+                }
+            }
             val newVariance =
                 if (it.variance == Variance.INVARIANT &&
                     p.variance != Variance.INVARIANT &&
                     // The next line forbids inferring a wildcard type when we have a corresponding Java type with conflicting variance.
                     // For example, Java might declare f(Comparable<CharSequence> cs), in which case we shouldn't add a `? super ...`
                     // wildcard. Note if javaType is unknown (e.g. this is a Kotlin source element), we assume wildcards should be added.
-                    (javaType?.let { jt -> jt is JavaWildcardType && jt.isExtends == (p.variance == Variance.OUT_VARIANCE) } != false) &&
-                    wildcardAdditionAllowed(p.variance, it.type, addByDefault))
+                    (javaVariance == null || javaVariance == p.variance) &&
+                    wildcardAdditionAllowed(p.variance, it.type, addByDefault, javaVariance))
                     p.variance
                 else
                     it.variance
@@ -1273,6 +1290,7 @@ open class KotlinUsesExtractor(
                         }
                         // Look for an exact type match...
                         javaClass.declarations.findSubType<IrFunction> { decl ->
+                            !decl.isFakeOverride &&
                             decl.name.asString() == jvmName &&
                             decl.valueParameters.size == f.valueParameters.size &&
                             decl.valueParameters.zip(f.valueParameters).all { p -> erase(p.first.type).classifierOrNull == erase(p.second.type).classifierOrNull }

java/kotlin-extractor/src/main/kotlin/LinesOfCode.kt

@@ -1,9 +1,10 @@
 package com.github.codeql
 
-import com.github.codeql.utils.versions.Psi2Ir
+import com.github.codeql.utils.versions.getPsi2Ir
 import com.intellij.psi.PsiComment
 import com.intellij.psi.PsiElement
 import com.intellij.psi.PsiWhiteSpace
+import org.jetbrains.kotlin.config.KotlinCompilerVersion
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.kdoc.psi.api.KDocElement
@@ -15,9 +16,16 @@ class LinesOfCode(
     val tw: FileTrapWriter,
     val file: IrFile
 ) {
-    val psi2Ir = Psi2Ir(logger)
+    val psi2Ir = getPsi2Ir(logger).also {
+        if (it == null) {
+            logger.warn("Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
+        }
+    }
 
     fun linesOfCodeInFile(id: Label<DbFile>) {
+        if (psi2Ir == null) {
+            return
+        }
         val ktFile = psi2Ir.getKtFile(file)
         if (ktFile == null) {
             return
@@ -26,6 +34,9 @@ class LinesOfCode(
     }
 
     fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label<out DbSourceline>) {
+        if (psi2Ir == null) {
+            return
+        }
         val p = psi2Ir.findPsiElement(d, file)
         if (p == null) {
             return

java/kotlin-extractor/src/main/kotlin/comments/CommentExtractor.kt

@@ -3,9 +3,11 @@ package com.github.codeql.comments
 import com.github.codeql.*
 import com.github.codeql.utils.IrVisitorLookup
 import com.github.codeql.utils.isLocalFunction
-import com.github.codeql.utils.versions.Psi2Ir
+import com.github.codeql.utils.versions.getPsi2Ir
+import com.github.codeql.utils.versions.Psi2IrFacade
 import com.intellij.psi.PsiComment
 import com.intellij.psi.PsiElement
+import org.jetbrains.kotlin.config.KotlinCompilerVersion
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.expressions.IrBody
@@ -21,18 +23,23 @@ import org.jetbrains.kotlin.psi.psiUtil.startOffset
 class CommentExtractor(private val fileExtractor: KotlinFileExtractor, private val file: IrFile, private val fileLabel: Label<out DbFile>) {
     private val tw = fileExtractor.tw
     private val logger = fileExtractor.logger
-    private val psi2Ir = Psi2Ir(logger)
-    private val ktFile = psi2Ir.getKtFile(file)
 
     fun extract() {
+        val psi2Ir = getPsi2Ir(logger)
+        if (psi2Ir == null) {
+            logger.warn("Comments will not be extracted as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
+            return
+        }
+        val ktFile = psi2Ir.getKtFile(file)
         if (ktFile == null) {
             logger.warn("Comments are not being processed in ${file.path}.")
-        } else {
-            ktFile.accept(commentVisitor)
+            return
         }
+        val commentVisitor = mkCommentVisitor(psi2Ir)
+        ktFile.accept(commentVisitor)
     }
 
-    private val commentVisitor =
+    private fun mkCommentVisitor(psi2Ir: Psi2IrFacade): KtVisitor<Unit, Unit> =
         object : KtVisitor<Unit, Unit>() {
             override fun visitElement(element: PsiElement) {
                 element.acceptChildren(this)

java/kotlin-extractor/src/main/kotlin/utils/IrVisitorLookup.kt

@@ -1,6 +1,6 @@
 package com.github.codeql.utils
 
-import com.github.codeql.utils.versions.Psi2Ir
+import com.github.codeql.utils.versions.Psi2IrFacade
 import com.intellij.psi.PsiElement
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.IrDeclaration
@@ -8,7 +8,7 @@ import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.util.isFakeOverride
 import org.jetbrains.kotlin.ir.visitors.IrElementVisitor
 
-class IrVisitorLookup(private val psi2Ir: Psi2Ir, private val psi: PsiElement, private val file: IrFile) :
+class IrVisitorLookup(private val psi2Ir: Psi2IrFacade, private val psi: PsiElement, private val file: IrFile) :
     IrElementVisitor<Unit, MutableCollection<IrElement>> {
     private val location = psi.getLocation()
 

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_4_32/Psi2Ir.kt

@@ -1,19 +1,5 @@
 package com.github.codeql.utils.versions
 
 import com.github.codeql.FileLogger
-import com.intellij.psi.PsiElement
-import org.jetbrains.kotlin.ir.IrElement
-import org.jetbrains.kotlin.ir.declarations.IrFile
-import org.jetbrains.kotlin.psi.KtFile
 
-class Psi2Ir(private val logger: FileLogger) : Psi2IrFacade {
-    override fun getKtFile(irFile: IrFile): KtFile? {
-        logger.warn("Comment extraction is not supported for Kotlin < 1.5.20")
-        return null
-    }
-
-    override fun findPsiElement(irElement: IrElement, irFile: IrFile): PsiElement? {
-        logger.error("Attempted comment extraction for Kotlin < 1.5.20")
-        return null
-    }
-}
+fun getPsi2Ir(@Suppress("UNUSED_PARAMETER") logger: FileLogger): Psi2IrFacade? = null

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_4_32/getKotlinType.kt

@@ -0,0 +1,6 @@
+package com.github.codeql.utils.versions
+
+import org.jetbrains.kotlin.ir.types.IrSimpleType
+import org.jetbrains.kotlin.ir.types.impl.IrTypeBase
+
+fun getKotlinType(s: IrSimpleType) = (s as? IrTypeBase)?.kotlinType

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_20/Psi2Ir.kt

@@ -8,7 +8,9 @@ import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.psi.KtFile
 
-class Psi2Ir(private val logger: FileLogger): Psi2IrFacade {
+fun getPsi2Ir(logger: FileLogger): Psi2IrFacade? = Psi2Ir(logger)
+
+private class Psi2Ir(private val logger: FileLogger): Psi2IrFacade {
     override fun getKtFile(irFile: IrFile): KtFile? {
         return irFile.getKtFile()
     }

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_7_0/getKotlinType.kt

@@ -0,0 +1,5 @@
+package com.github.codeql.utils.versions
+
+import org.jetbrains.kotlin.ir.types.IrSimpleType
+
+fun getKotlinType(s: IrSimpleType) = s.kotlinType

java/old-change-notes/2021-02-23-deprecated-jcenter-bintray.md

@@ -1,4 +1,4 @@
 lgtm,codescanning
 * A new query "Depending upon JCenter/Bintray as an artifact repository" 
   (`java/maven/dependency-upon-bintray`) has been added.
-  This query finds uses of the deprecated JCenter/Bintray artifact respositories.
+  This query finds uses of the deprecated JCenter/Bintray artifact repositories.

java/ql/consistency-queries/diags.ql

@@ -0,0 +1,34 @@
+import semmle.code.java.Diagnostics
+
+/*
+ * This query fails if any unexpected diagnostics are recorded in the
+ * database. By putting
+ *    // Diagnostic Matches: PAT
+ * in any source files, you can declare that diagnostics matching PAT
+ * (in the string.matches(string) sense) are expected.
+ */
+
+class DiagnosticException extends Top {
+  string pattern;
+
+  DiagnosticException() {
+    this.(KtComment).getText() = "// Diagnostic Matches: " + pattern
+    or
+    this.(Javadoc).toString() = "// Diagnostic Matches: " + pattern
+  }
+
+  Diagnostic getException() { diagnosticMessage(result).matches(pattern) }
+}
+
+string diagnosticMessage(Diagnostic d) {
+  if d.getFullMessage() != "" then result = d.getFullMessage() else result = d.getMessage()
+}
+
+// Check that there aren't any old DiagnosticExceptions left after
+// something is fixed.
+query predicate unusedDiagnosticException(DiagnosticException de) { not exists(de.getException()) }
+
+query predicate unexpectedDiagnostic(Diagnostic d, string s) {
+  s = diagnosticMessage(d) and
+  not d = any(DiagnosticException de).getException()
+}

java/ql/consistency-queries/javaEquivalent.ql

@@ -1,6 +0,0 @@
-import java
-import semmle.code.java.Diagnostics
-
-from Diagnostic d
-where exists(d.getMessage().indexOf("Couldn't find a Java equivalent function to "))
-select d

java/ql/consistency-queries/qlpack.yml

@@ -1,4 +1,4 @@
 name: codeql-java-consistency-queries
 version: 0.0.0
-libraryPathDependencies:
-  - codeql-java
+dependencies:
+  codeql/java-all: '*'

java/ql/integration-tests/linux-only/kotlin/custom_plugin/diagnostics.expected

@@ -1,3 +1 @@
 | CodeQL Kotlin extractor | 2 |  | IrProperty without a getter | d.kt:0:0:0:0 | d.kt:0:0:0:0 |
-| CodeQL Kotlin extractor | 2 |  | Not rewriting trap file for test-db/trap/java/classes/java/lang/Boolean.members/Boolean.members<VERSION>-<MODIFIED>-kotlin.trap.gz as it exists | file://:0:0:0:0 | file://:0:0:0:0 |
-| CodeQL Kotlin extractor | 2 |  | Not rewriting trap file for test-db/trap/java/classes/kotlin/Boolean.members/Boolean.members<VERSION>-<MODIFIED>-null.trap.gz as it exists | file://:0:0:0:0 | file://:0:0:0:0 |

java/ql/integration-tests/linux-only/kotlin/qlpack.yml

@@ -1,2 +1,2 @@
-libraryPathDependencies:
-  - codeql-java
+dependencies:
+  codeql/java-all: '*'

java/ql/integration-tests/posix-only/kotlin/default-parameter-mad-flow/lib.kt

@@ -0,0 +1,35 @@
+class ConstructorWithDefaults(x: Int, y: Int = 1) { }
+
+fun topLevelWithDefaults(x: Int, y: Int = 1) = 0
+fun String.extensionWithDefaults(x: Int, y: Int = 1) = 0
+
+class LibClass {
+
+  fun memberWithDefaults(x: Int, y: Int = 1) = 0
+  fun String.extensionMemberWithDefaults(x: Int, y: Int = 1) = 0
+
+  fun multiParameterTest(x: Int, y: Int, z: Int, w: Int = 0) = 0
+  fun Int.multiParameterExtensionTest(x: Int, y: Int, w: Int = 0) = 0
+
+}
+
+class SomeToken {}
+
+fun topLevelArgSource(st: SomeToken, x: Int = 0) {}
+fun String.extensionArgSource(st: SomeToken, x: Int = 0) {}
+
+class SourceClass {
+
+  fun memberArgSource(st: SomeToken, x: Int = 0) {}
+
+}
+
+fun topLevelSink(x: Int, y: Int = 1) {}
+fun String.extensionSink(x: Int, y: Int = 1) {}
+
+class SinkClass(x: Int, y: Int = 1) {
+
+  fun memberSink(x: Int, y: Int = 1) {}
+  fun String.extensionMemberSink(x: Int, y: Int = 1) {}
+
+}

java/ql/integration-tests/posix-only/kotlin/default-parameter-mad-flow/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+import subprocess
+
+subprocess.check_call(["kotlinc", "lib.kt", "-d", "lib"])
+run_codeql_database_create(["kotlinc user.kt -cp lib"], lang="java")

java/ql/integration-tests/posix-only/kotlin/default-parameter-mad-flow/test.ql

@@ -0,0 +1,74 @@
+import java
+import semmle.code.java.dataflow.TaintTracking
+import TestUtilities.InlineExpectationsTest
+private import semmle.code.java.dataflow.ExternalFlow
+
+private class Models extends SummaryModelCsv {
+  override predicate row(string row) {
+    row =
+      [
+        ";ConstructorWithDefaults;true;ConstructorWithDefaults;(int,int);;Argument[0];Argument[-1];taint;manual",
+        ";LibKt;true;topLevelWithDefaults;(int,int);;Argument[0];ReturnValue;value;manual",
+        ";LibKt;true;extensionWithDefaults;(String,int,int);;Argument[1];ReturnValue;value;manual",
+        ";LibClass;true;memberWithDefaults;(int,int);;Argument[0];ReturnValue;value;manual",
+        ";LibClass;true;extensionMemberWithDefaults;(String,int,int);;Argument[1];ReturnValue;value;manual",
+        ";LibClass;true;multiParameterTest;(int,int,int,int);;Argument[0..1];ReturnValue;value;manual",
+        ";LibClass;true;multiParameterExtensionTest;(int,int,int,int);;Argument[0, 1];ReturnValue;value;manual",
+      ]
+  }
+}
+
+private class SourceModels extends SourceModelCsv {
+  override predicate row(string row) {
+    row =
+      [
+        ";LibKt;true;topLevelArgSource;(SomeToken,int);;Argument[0];kotlinMadFlowTest;manual",
+        ";LibKt;true;extensionArgSource;(String,SomeToken,int);;Argument[1];kotlinMadFlowTest;manual",
+        ";SourceClass;true;memberArgSource;(SomeToken,int);;Argument[0];kotlinMadFlowTest;manual"
+      ]
+  }
+}
+
+private class SinkModels extends SinkModelCsv {
+  override predicate row(string row) {
+    row =
+      [
+        ";SinkClass;true;SinkClass;(int,int);;Argument[0];kotlinMadFlowTest;manual",
+        ";LibKt;true;topLevelSink;(int,int);;Argument[0];kotlinMadFlowTest;manual",
+        ";LibKt;true;extensionSink;(String,int,int);;Argument[1];kotlinMadFlowTest;manual",
+        ";SinkClass;true;memberSink;(int,int);;Argument[0];kotlinMadFlowTest;manual",
+        ";SinkClass;true;extensionMemberSink;(String,int,int);;Argument[1];kotlinMadFlowTest;manual"
+      ]
+  }
+}
+
+class Config extends TaintTracking::Configuration {
+  Config() { this = "Config" }
+
+  override predicate isSource(DataFlow::Node n) {
+    n.asExpr().(MethodAccess).getCallee().getName() = "source"
+    or
+    sourceNode(n, "kotlinMadFlowTest")
+  }
+
+  override predicate isSink(DataFlow::Node n) {
+    n.asExpr().(Argument).getCall().getCallee().getName() = "sink"
+    or
+    sinkNode(n, "kotlinMadFlowTest")
+  }
+}
+
+class InlineFlowTest extends InlineExpectationsTest {
+  InlineFlowTest() { this = "HasFlowTest" }
+
+  override string getARelevantTag() { result = "flow" }
+
+  override predicate hasActualResult(Location location, string element, string tag, string value) {
+    tag = "flow" and
+    exists(DataFlow::Node src, DataFlow::Node sink, Config c | c.hasFlow(src, sink) |
+      sink.getLocation() = location and
+      element = sink.toString() and
+      value = ""
+    )
+  }
+}

java/ql/integration-tests/posix-only/kotlin/default-parameter-mad-flow/user.kt

@@ -0,0 +1,60 @@
+fun source() = 1
+
+fun sink(x: Any) { }
+
+fun test(c: LibClass, sourcec: SourceClass, sinkc: SinkClass) {
+
+  sink(ConstructorWithDefaults(source(), 0)) // $ flow
+  sink(ConstructorWithDefaults(source())) // $ flow
+
+  sink(topLevelWithDefaults(source(), 0)) // $ flow
+  sink(topLevelWithDefaults(source())) // $ flow
+
+  sink("Hello world".extensionWithDefaults(source(), 0)) // $ flow
+  sink("Hello world".extensionWithDefaults(source())) // $ flow
+
+  sink(c.memberWithDefaults(source(), 0)) // $ flow
+  sink(c.memberWithDefaults(source())) // $ flow
+
+  sink(c.multiParameterTest(source(), 0, 0)) // $ flow
+  sink(c.multiParameterTest(0, source(), 0)) // $ flow
+  sink(c.multiParameterTest(0, 0, source()))
+
+  with(c) {
+    sink("Hello world".extensionMemberWithDefaults(source(), 0)) // $ flow
+    sink("Hello world".extensionMemberWithDefaults(source())) // $ flow
+  }
+
+  with(c) {
+    sink(source().multiParameterExtensionTest(0, 0)) // $ flow
+    sink(0.multiParameterExtensionTest(source(), 0)) // $ flow
+    sink(0.multiParameterExtensionTest(0, source()))
+  }
+
+  run {
+    val st = SomeToken()
+    topLevelArgSource(st)
+    sink(st) // $ flow
+  }
+
+  run {
+    val st = SomeToken()
+    "Hello world".extensionArgSource(st)
+    sink(st) // $ flow
+  }
+
+  run {
+    val st = SomeToken()
+    sourcec.memberArgSource(st)
+    sink(st) // $ flow
+  }
+
+  SinkClass(source()) // $ flow
+  topLevelSink(source()) // $ flow
+  "Hello world".extensionSink(source()) // $ flow
+  sinkc.memberSink(source()) // $ flow
+  with(sinkc) {
+    "Hello world".extensionMemberSink(source()) // $ flow
+  }
+
+}

java/ql/integration-tests/posix-only/kotlin/enhanced-nullability/NotNull.java

@@ -0,0 +1,6 @@
+package org.jetbrains.annotations;
+import java.lang.annotation.*;
+
+// Stub of @NotNull:
+@Target({ElementType.METHOD, ElementType.FIELD, ElementType.PARAMETER, ElementType.LOCAL_VARIABLE, ElementType.TYPE_USE})
+public @interface NotNull { }

java/ql/integration-tests/posix-only/kotlin/enhanced-nullability/Test.java

@@ -0,0 +1,7 @@
+import org.jetbrains.annotations.NotNull;
+
+public class Test {
+
+  public @NotNull Integer f(@NotNull Integer p) { return p; }
+
+}

java/ql/integration-tests/posix-only/kotlin/enhanced-nullability/test.expected

@@ -0,0 +1,14 @@
+exprs
+| Test.java:5:19:5:25 | Integer | Integer |
+| Test.java:5:38:5:44 | Integer | Integer |
+| Test.java:5:58:5:58 | p | Integer |
+| user.kt:2:3:2:16 | x | int |
+| user.kt:2:11:2:11 | t | Test |
+| user.kt:2:13:2:16 | <implicit not null> | int |
+| user.kt:2:13:2:16 | f(...) | Integer |
+| user.kt:2:13:2:16 | int | int |
+| user.kt:2:15:2:15 | 5 | int |
+| user.kt:3:10:3:10 | x | int |
+#select
+| Test.java:5:27:5:27 | f | Integer |
+| user.kt:1:1:4:1 | f | Test |

java/ql/integration-tests/posix-only/kotlin/enhanced-nullability/test.py

@@ -0,0 +1,6 @@
+from create_database_utils import *
+import glob
+
+os.mkdir('build')
+runSuccessfully(["javac"] + glob.glob("*.java") + ["-d", "build"])
+run_codeql_database_create(["javac " + " ".join(glob.glob("*.java")) + " -d build", "kotlinc user.kt -cp build"], lang="java")

java/ql/integration-tests/posix-only/kotlin/enhanced-nullability/test.ql

@@ -0,0 +1,9 @@
+import java
+
+query predicate exprs(Expr e, string t) {
+  e.getEnclosingCallable().getDeclaringType().fromSource() and t = e.getType().toString()
+}
+
+from Method m
+where m.fromSource()
+select m, m.getAParamType().toString()

java/ql/integration-tests/posix-only/kotlin/enhanced-nullability/user.kt

@@ -0,0 +1,4 @@
+fun f(t: Test): Int {
+  val x = t.f(5)
+  return x
+}

java/ql/integration-tests/posix-only/kotlin/external-property-overloads/test.expected

@@ -0,0 +1,2 @@
+| user.kt:3:22:3:22 | getF(...) | lib/lib/TestKt.class:0:0:0:0 | getF |
+| user.kt:3:28:3:28 | getF(...) | lib/lib/TestKt.class:0:0:0:0 | getF |

java/ql/integration-tests/posix-only/kotlin/external-property-overloads/test.kt

@@ -0,0 +1,7 @@
+package lib
+
+val String.f
+  get() = 1
+
+val Int.f
+  get() = 2

java/ql/integration-tests/posix-only/kotlin/external-property-overloads/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+
+os.mkdir('lib')
+runSuccessfully(["kotlinc", "test.kt", "-d", "lib"])
+run_codeql_database_create(["kotlinc user.kt -cp lib"], lang="java")

java/ql/integration-tests/posix-only/kotlin/external-property-overloads/test.ql

@@ -0,0 +1,4 @@
+import java
+
+from MethodAccess ma
+select ma, ma.getCallee()

java/ql/integration-tests/posix-only/kotlin/external-property-overloads/user.kt

@@ -0,0 +1,3 @@
+import lib.f
+
+fun test() = "hello".f + 1.f

java/ql/integration-tests/posix-only/kotlin/gradle_kotlinx_serialization/ConstantExpAppearsNonConstant.qlref

@@ -0,0 +1 @@
+Likely Bugs/Arithmetic/ConstantExpAppearsNonConstant.ql

java/ql/integration-tests/posix-only/kotlin/gradle_kotlinx_serialization/PrintAst.expected

@@ -478,3 +478,296 @@ app/src/main/kotlin/testProject/App.kt:
 #    8|     16: [FieldDeclaration] int language;
 #    8|       -1: [TypeAccess] int
 #    8|       0: [VarAccess] language
+#   10|   2: [Interface] Base
+#   11|     1: [Method] getId
+#   11|       3: [TypeAccess] String
+#   14|   3: [Class] X
+#    0|     1: [Constructor] X
+#-----|       4: (Parameters)
+#    0|         0: [Parameter] seen1
+#    0|           0: [TypeAccess] int
+#    0|         1: [Parameter] id
+#    0|           0: [TypeAccess] String
+#    0|         2: [Parameter] serializationConstructorMarker
+#    0|           0: [TypeAccess] SerializationConstructorMarker
+#   14|       5: [BlockStmt] { ... }
+#   14|         0: [ExprStmt] <Expr>;
+#   14|           0: [WhenExpr] when ...
+#   14|             0: [WhenBranch] ... -> ...
+#   14|               0: [ValueNEExpr] ... (value not-equals) ...
+#   14|                 0: [IntegerLiteral] 0
+#   14|                 1: [AndBitwiseExpr] ... & ...
+#   14|                   0: [IntegerLiteral] 0
+#   14|                   1: [VarAccess] seen1
+#   14|               1: [ExprStmt] <Expr>;
+#   14|                 0: [MethodAccess] throwMissingFieldException(...)
+#   14|                   -1: [TypeAccess] PluginExceptionsKt
+#   14|                   0: [VarAccess] seen1
+#   14|                   1: [IntegerLiteral] 0
+#   14|                   2: [MethodAccess] getDescriptor(...)
+#   14|                     -1: [VarAccess] INSTANCE
+#   14|         1: [SuperConstructorInvocationStmt] super(...)
+#   14|         2: [ExprStmt] <Expr>;
+#   14|           0: [WhenExpr] when ...
+#   14|             0: [WhenBranch] ... -> ...
+#   14|               0: [ValueEQExpr] ... (value equals) ...
+#   14|                 0: [IntegerLiteral] 0
+#   14|                 1: [AndBitwiseExpr] ... & ...
+#   14|                   0: [VarAccess] seen1
+#   14|                   1: [IntegerLiteral] 1
+#   14|               1: [ExprStmt] <Expr>;
+#   14|                 0: [AssignExpr] ...=...
+#   14|                   0: [VarAccess] X.this.id
+#   14|                     -1: [ThisAccess] X.this
+#   14|                       0: [TypeAccess] X
+#   16|                   1: [StringLiteral] X
+#   14|             1: [WhenBranch] ... -> ...
+#   14|               0: [BooleanLiteral] true
+#   14|               1: [ExprStmt] <Expr>;
+#   14|                 0: [AssignExpr] ...=...
+#   14|                   0: [VarAccess] X.this.id
+#   14|                     -1: [ThisAccess] X.this
+#   14|                       0: [TypeAccess] X
+#   14|                   1: [VarAccess] id
+#    0|     2: [Method] write$Self
+#    0|       3: [TypeAccess] Unit
+#-----|       4: (Parameters)
+#    0|         0: [Parameter] self
+#    0|           0: [TypeAccess] X
+#    0|         1: [Parameter] output
+#    0|           0: [TypeAccess] CompositeEncoder
+#    0|         2: [Parameter] serialDesc
+#    0|           0: [TypeAccess] SerialDescriptor
+#   14|       5: [BlockStmt] { ... }
+#   14|         0: [ExprStmt] <Expr>;
+#   14|           0: [WhenExpr] when ...
+#   14|             0: [WhenBranch] ... -> ...
+#   14|               0: [WhenExpr] when ...
+#   14|                 0: [WhenBranch] ... -> ...
+#   14|                   0: [MethodAccess] shouldEncodeElementDefault(...)
+#   14|                     -1: [VarAccess] output
+#   14|                     0: [VarAccess] serialDesc
+#   14|                     1: [IntegerLiteral] 0
+#   14|                   1: [ExprStmt] <Expr>;
+#   14|                     0: [BooleanLiteral] true
+#   14|                 1: [WhenBranch] ... -> ...
+#   14|                   0: [BooleanLiteral] true
+#   14|                   1: [ExprStmt] <Expr>;
+#   14|                     0: [ValueNEExpr] ... (value not-equals) ...
+#   14|                       0: [MethodAccess] getId(...)
+#   14|                         -1: [VarAccess] self
+#   16|                       1: [StringLiteral] X
+#   14|               1: [ExprStmt] <Expr>;
+#   14|                 0: [MethodAccess] encodeStringElement(...)
+#   14|                   -1: [VarAccess] output
+#   14|                   0: [VarAccess] serialDesc
+#   14|                   1: [IntegerLiteral] 0
+#   14|                   2: [MethodAccess] getId(...)
+#   14|                     -1: [VarAccess] self
+#   14|     3: [Class] $serializer
+#    0|       1: [FieldDeclaration] SerialDescriptor descriptor;
+#    0|         -1: [TypeAccess] SerialDescriptor
+#    0|       2: [Method] childSerializers
+#    0|         3: [TypeAccess] KSerializer<?>[]
+#    0|           0: [TypeAccess] KSerializer<?>
+#    0|             0: [WildcardTypeAccess] ? ...
+#   14|         5: [BlockStmt] { ... }
+#   14|           0: [ReturnStmt] return ...
+#   14|             0: [ArrayCreationExpr] new KSerializer<?>[]
+#   14|               -2: [ArrayInit] {...}
+#   14|               -1: [TypeAccess] KSerializer<?>
+#   14|               0: [IntegerLiteral] 1
+#    0|       3: [Method] deserialize
+#    0|         3: [TypeAccess] X
+#-----|         4: (Parameters)
+#    0|           0: [Parameter] decoder
+#    0|             0: [TypeAccess] Decoder
+#   14|         5: [BlockStmt] { ... }
+#   14|           0: [LocalVariableDeclStmt] var ...;
+#   14|             1: [LocalVariableDeclExpr] tmp0_desc
+#   14|               0: [MethodAccess] getDescriptor(...)
+#   14|                 -1: [ThisAccess] this
+#   14|           1: [LocalVariableDeclStmt] var ...;
+#   14|             1: [LocalVariableDeclExpr] tmp1_flag
+#   14|               0: [BooleanLiteral] true
+#   14|           2: [LocalVariableDeclStmt] var ...;
+#   14|             1: [LocalVariableDeclExpr] tmp2_index
+#   14|               0: [IntegerLiteral] 0
+#   14|           3: [LocalVariableDeclStmt] var ...;
+#   14|             1: [LocalVariableDeclExpr] tmp3_bitMask0
+#   14|               0: [IntegerLiteral] 0
+#   14|           4: [LocalVariableDeclStmt] var ...;
+#   14|             1: [LocalVariableDeclExpr] tmp4_local0
+#   14|               0: [NullLiteral] null
+#   14|           5: [LocalVariableDeclStmt] var ...;
+#   14|             1: [LocalVariableDeclExpr] tmp5_input
+#   14|               0: [MethodAccess] beginStructure(...)
+#   14|                 -1: [VarAccess] decoder
+#   14|                 0: [VarAccess] tmp0_desc
+#   14|           6: [ExprStmt] <Expr>;
+#   14|             0: [WhenExpr] when ...
+#   14|               0: [WhenBranch] ... -> ...
+#   14|                 0: [MethodAccess] decodeSequentially(...)
+#   14|                   -1: [VarAccess] tmp5_input
+#   14|                 1: [BlockStmt] { ... }
+#   14|                   0: [BlockStmt] { ... }
+#   14|                     0: [ExprStmt] <Expr>;
+#   14|                       0: [AssignExpr] ...=...
+#   14|                         0: [VarAccess] tmp4_local0
+#   14|                         1: [MethodAccess] decodeStringElement(...)
+#   14|                           -1: [VarAccess] tmp5_input
+#   14|                           0: [VarAccess] tmp0_desc
+#   14|                           1: [IntegerLiteral] 0
+#   14|                     1: [ExprStmt] <Expr>;
+#   14|                       0: [AssignExpr] ...=...
+#   14|                         0: [VarAccess] tmp3_bitMask0
+#   14|                         1: [OrBitwiseExpr] ... | ...
+#   14|                           0: [VarAccess] tmp3_bitMask0
+#   14|                           1: [IntegerLiteral] 1
+#   14|               1: [WhenBranch] ... -> ...
+#   14|                 0: [BooleanLiteral] true
+#   14|                 1: [WhileStmt] while (...)
+#   14|                   0: [VarAccess] tmp1_flag
+#   14|                   1: [BlockStmt] { ... }
+#   14|                     0: [ExprStmt] <Expr>;
+#   14|                       0: [AssignExpr] ...=...
+#   14|                         0: [VarAccess] tmp2_index
+#   14|                         1: [MethodAccess] decodeElementIndex(...)
+#   14|                           -1: [VarAccess] tmp5_input
+#   14|                           0: [VarAccess] tmp0_desc
+#   14|                     1: [ExprStmt] <Expr>;
+#   14|                       0: [WhenExpr] when ...
+#   14|                         0: [WhenBranch] ... -> ...
+#   14|                           0: [ValueEQExpr] ... (value equals) ...
+#   14|                             0: [VarAccess] tmp2_index
+#   14|                             1: [IntegerLiteral] -1
+#   14|                           1: [ExprStmt] <Expr>;
+#   14|                             0: [AssignExpr] ...=...
+#   14|                               0: [VarAccess] tmp1_flag
+#   14|                               1: [BooleanLiteral] false
+#   14|                         1: [WhenBranch] ... -> ...
+#   14|                           0: [ValueEQExpr] ... (value equals) ...
+#   14|                             0: [VarAccess] tmp2_index
+#   14|                             1: [IntegerLiteral] 0
+#   14|                           1: [BlockStmt] { ... }
+#   14|                             0: [ExprStmt] <Expr>;
+#   14|                               0: [AssignExpr] ...=...
+#   14|                                 0: [VarAccess] tmp4_local0
+#   14|                                 1: [MethodAccess] decodeStringElement(...)
+#   14|                                   -1: [VarAccess] tmp5_input
+#   14|                                   0: [VarAccess] tmp0_desc
+#   14|                                   1: [IntegerLiteral] 0
+#   14|                             1: [ExprStmt] <Expr>;
+#   14|                               0: [AssignExpr] ...=...
+#   14|                                 0: [VarAccess] tmp3_bitMask0
+#   14|                                 1: [OrBitwiseExpr] ... | ...
+#   14|                                   0: [VarAccess] tmp3_bitMask0
+#   14|                                   1: [IntegerLiteral] 1
+#   14|                         2: [WhenBranch] ... -> ...
+#   14|                           0: [BooleanLiteral] true
+#   14|                           1: [ThrowStmt] throw ...
+#   14|                             0: [ClassInstanceExpr] new UnknownFieldException(...)
+#   14|                               -3: [TypeAccess] UnknownFieldException
+#   14|                               0: [VarAccess] tmp2_index
+#   14|           7: [ExprStmt] <Expr>;
+#   14|             0: [MethodAccess] endStructure(...)
+#   14|               -1: [VarAccess] tmp5_input
+#   14|               0: [VarAccess] tmp0_desc
+#   14|           8: [ReturnStmt] return ...
+#   14|             0: [ClassInstanceExpr] new X(...)
+#   14|               -3: [TypeAccess] X
+#   14|               0: [VarAccess] tmp3_bitMask0
+#   14|               1: [VarAccess] tmp4_local0
+#   14|               2: [NullLiteral] null
+#    0|       4: [Method] getDescriptor
+#    0|         3: [TypeAccess] SerialDescriptor
+#    0|         5: [BlockStmt] { ... }
+#    0|           0: [ReturnStmt] return ...
+#    0|             0: [VarAccess] this.descriptor
+#    0|               -1: [ThisAccess] this
+#    0|       5: [Method] serialize
+#    0|         3: [TypeAccess] Unit
+#-----|         4: (Parameters)
+#    0|           0: [Parameter] encoder
+#    0|             0: [TypeAccess] Encoder
+#    0|           1: [Parameter] value
+#    0|             0: [TypeAccess] X
+#   14|         5: [BlockStmt] { ... }
+#   14|           0: [LocalVariableDeclStmt] var ...;
+#   14|             1: [LocalVariableDeclExpr] tmp0_desc
+#   14|               0: [MethodAccess] getDescriptor(...)
+#   14|                 -1: [ThisAccess] this
+#   14|           1: [LocalVariableDeclStmt] var ...;
+#   14|             1: [LocalVariableDeclExpr] tmp1_output
+#   14|               0: [MethodAccess] beginStructure(...)
+#   14|                 -1: [VarAccess] encoder
+#   14|                 0: [VarAccess] tmp0_desc
+#   14|           2: [ExprStmt] <Expr>;
+#   14|             0: [MethodAccess] write$Self(...)
+#   14|               -1: [TypeAccess] X
+#   14|               0: [VarAccess] value
+#   14|               1: [VarAccess] tmp1_output
+#   14|               2: [VarAccess] tmp0_desc
+#   14|           3: [ExprStmt] <Expr>;
+#   14|             0: [MethodAccess] endStructure(...)
+#   14|               -1: [VarAccess] tmp1_output
+#   14|               0: [VarAccess] tmp0_desc
+#   14|       6: [Constructor] $serializer
+#   14|         5: [BlockStmt] { ... }
+#   14|           0: [SuperConstructorInvocationStmt] super(...)
+#   14|           1: [BlockStmt] { ... }
+#   14|             0: [LocalVariableDeclStmt] var ...;
+#   14|               1: [LocalVariableDeclExpr] tmp0_serialDesc
+#   14|                 0: [ClassInstanceExpr] new PluginGeneratedSerialDescriptor(...)
+#   14|                   -3: [TypeAccess] PluginGeneratedSerialDescriptor
+#   14|                   0: [StringLiteral] testProject.X
+#   14|                   1: [ThisAccess] $serializer.this
+#   14|                     0: [TypeAccess] $serializer
+#   14|                   2: [IntegerLiteral] 1
+#   14|             1: [ExprStmt] <Expr>;
+#   14|               0: [MethodAccess] addElement(...)
+#   14|                 -1: [VarAccess] tmp0_serialDesc
+#   14|                 0: [StringLiteral] id
+#   14|                 1: [BooleanLiteral] true
+#   14|             2: [ExprStmt] <Expr>;
+#   14|               0: [AssignExpr] ...=...
+#   14|                 0: [VarAccess] $serializer.this.descriptor
+#   14|                   -1: [ThisAccess] $serializer.this
+#   14|                     0: [TypeAccess] $serializer
+#   14|                 1: [VarAccess] tmp0_serialDesc
+#   14|       7: [Method] typeParametersSerializers
+#   14|         3: [TypeAccess] KSerializer<?>[]
+#   14|           0: [TypeAccess] KSerializer<?>
+#   14|             0: [WildcardTypeAccess] ? ...
+#   14|         5: [BlockStmt] { ... }
+#   14|           0: [ReturnStmt] return ...
+#   14|             0: [MethodAccess] typeParametersSerializers(...)
+#   14|               -1: [SuperAccess] GeneratedSerializer.super
+#   14|                 0: [TypeAccess] GeneratedSerializer
+#   14|     4: [Class] Companion
+#    0|       1: [Method] serializer
+#    0|         3: [TypeAccess] KSerializer<X>
+#    0|           0: [TypeAccess] X
+#   14|         5: [BlockStmt] { ... }
+#   14|           0: [ReturnStmt] return ...
+#   14|             0: [VarAccess] INSTANCE
+#   14|       2: [Constructor] Companion
+#   14|         5: [BlockStmt] { ... }
+#   14|           0: [SuperConstructorInvocationStmt] super(...)
+#   14|           1: [BlockStmt] { ... }
+#   15|     5: [Constructor] X
+#   14|       5: [BlockStmt] { ... }
+#   14|         0: [SuperConstructorInvocationStmt] super(...)
+#   15|         1: [BlockStmt] { ... }
+#   16|           0: [ExprStmt] <Expr>;
+#   16|             0: [KtInitializerAssignExpr] ...=...
+#   16|               0: [VarAccess] id
+#   16|     6: [FieldDeclaration] String id;
+#   16|       -1: [TypeAccess] String
+#   16|       0: [StringLiteral] X
+#   16|     7: [Method] getId
+#   16|       3: [TypeAccess] String
+#   16|       5: [BlockStmt] { ... }
+#   16|         0: [ReturnStmt] return ...
+#   16|           0: [VarAccess] this.id
+#   16|             -1: [ThisAccess] this

java/ql/integration-tests/posix-only/kotlin/gradle_kotlinx_serialization/app/src/main/kotlin/testProject/App.kt

@@ -5,4 +5,13 @@ import kotlinx.serialization.json.*
 import kotlinx.serialization.Serializable
 
 @Serializable
-data class Project(val name: String, val language: Int)
+data class Project(val name: String, val language: Int)
+
+interface Base {
+    val id: String
+}
+
+@Serializable
+class X private constructor() : Base {
+    override val id: String = "X"
+}

java/ql/integration-tests/posix-only/kotlin/gradle_kotlinx_serialization/diagnostics.expected

@@ -1 +1,2 @@
 | CodeQL Kotlin extractor | 5 |  | Unbound symbol found, skipping extraction of expression | app/src/main/kotlin/testProject/App.kt:7:1:8:55 | app/src/main/kotlin/testProject/App.kt:7:1:8:55 |
+| CodeQL Kotlin extractor | 5 |  | Unbound symbol found, skipping extraction of expression | app/src/main/kotlin/testProject/App.kt:14:1:17:1 | app/src/main/kotlin/testProject/App.kt:14:1:17:1 |

java/ql/integration-tests/posix-only/kotlin/java-interface-redeclares-tostring/Test.java

@@ -0,0 +1,5 @@
+public interface Test {
+  String toString();
+  int hashCode();
+  boolean equals(Object other);
+}

java/ql/integration-tests/posix-only/kotlin/java-interface-redeclares-tostring/test.expected

@@ -0,0 +1,4 @@
+| equals | Test |
+| hashCode | Test |
+| toString | Test |
+| toString | java.lang.CharSequence |

java/ql/integration-tests/posix-only/kotlin/java-interface-redeclares-tostring/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+
+os.mkdir('bin')
+runSuccessfully(["javac", "Test.java", "-d", "bin"])
+run_codeql_database_create(["kotlinc user.kt -cp bin"], lang="java")

java/ql/integration-tests/posix-only/kotlin/java-interface-redeclares-tostring/test.ql

@@ -0,0 +1,7 @@
+import java
+
+from Method m
+where
+  m.getDeclaringType().getName() = ["Test", "CharSequence"] and
+  m.getName() = ["toString", "equals", "hashCode"]
+select m.getName(), m.getDeclaringType().getQualifiedName()

java/ql/integration-tests/posix-only/kotlin/java-interface-redeclares-tostring/user.kt

@@ -0,0 +1 @@
+fun f(t: Test, cs: CharSequence) = t.toString() + cs.toString() + t.equals(1) + t.hashCode()

java/ql/integration-tests/posix-only/kotlin/logs/index_logs.py

@@ -32,6 +32,7 @@ with open('logs.csv', 'w', newline='') as f_out:
                     j = json.loads(line)
                     msg = j['message']
                     msg = re.sub('(?<=Extraction for invocation TRAP file ).*/kt-db/trap/java/invocations/kotlin\..*\.trap', '<FILENAME>', msg)
+                    msg = re.sub('(?<=Kotlin version )[0-9.]+', '<VERSION>', msg)
                     if msg.startswith('Peak memory: '):
                         # Peak memory information varies from run to run, so just ignore it
                         continue

java/ql/integration-tests/posix-only/kotlin/logs/logs.expected

@@ -1,5 +1,6 @@
 | 1 | 1 | Test script | Log file | 1 |
 | 1 | 2 | CodeQL Kotlin extractor | INFO | Extraction started |
 | 1 | 3 | CodeQL Kotlin extractor | INFO | Extraction for invocation TRAP file <FILENAME> |
-| 1 | 4 | CodeQL Kotlin extractor | INFO | Extracting file test.kt |
-| 1 | 5 | CodeQL Kotlin extractor | INFO | Extraction completed |
+| 1 | 4 | CodeQL Kotlin extractor | INFO | Kotlin version <VERSION> |
+| 1 | 5 | CodeQL Kotlin extractor | INFO | Extracting file test.kt |
+| 1 | 6 | CodeQL Kotlin extractor | INFO | Extraction completed |

java/ql/integration-tests/posix-only/kotlin/needless-java-wildcards/Test.java

@@ -0,0 +1,9 @@
+public class Test {
+
+  // This gets mapped to kotlin.Iterable<out T>, meaning we must reintroduce the use-site extends variance to get a type consistent with Java.
+  public static void needlessExtends(Iterable<? extends String> l) { }
+
+  // This type is defined KotlinConsumer<in T>, meaning we must reintroduce the use-site extends variance to get a type consistent with Java.
+  public static void needlessSuper(KotlinConsumer<? super Object> l) { }
+
+}

java/ql/integration-tests/posix-only/kotlin/needless-java-wildcards/kConsumer.kt

@@ -0,0 +1 @@
+public class KotlinConsumer<in T> { }

java/ql/integration-tests/posix-only/kotlin/needless-java-wildcards/test.expected

@@ -0,0 +1,2 @@
+| Test.java:4:22:4:36 | needlessExtends | file://<external>/Iterable.class:0:0:0:0 | Iterable<? extends String> |
+| Test.java:7:22:7:34 | needlessSuper | build1/KotlinConsumer.class:0:0:0:0 | KotlinConsumer<? super Object> |

java/ql/integration-tests/posix-only/kotlin/needless-java-wildcards/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+
+os.mkdir('build1')
+os.mkdir('build2')
+run_codeql_database_create(["kotlinc kConsumer.kt -d build1", "javac Test.java -cp build1 -d build2", "kotlinc user.kt -cp build1:build2"], lang="java")

java/ql/integration-tests/posix-only/kotlin/needless-java-wildcards/test.ql

@@ -0,0 +1,15 @@
+import java
+
+class ClassOrInterfaceLocation extends ClassOrInterface {
+  override predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
+    exists(string fullPath | super.hasLocationInfo(fullPath, sl, sc, el, ec) |
+      if exists(this.getFile().getRelativePath())
+      then path = fullPath
+      else path = fullPath.regexpReplaceAll(".*/", "<external>/")
+    )
+  }
+}
+
+from Method m
+where m.fromSource()
+select m, m.getAParamType()

java/ql/integration-tests/posix-only/kotlin/needless-java-wildcards/user.kt

@@ -0,0 +1,4 @@
+fun f() {
+  Test.needlessExtends(null)
+  Test.needlessSuper(null)
+}