Subsample sinks before scoring to avoid DCA timeouts

Locations only exist where there are locatable structures in the DB. Thus, select the largest location that contains the node and at most `neighborhoodSize` lines before and after the node.

.github/actions/find-latest-bundle/action.yml

@@ -0,0 +1,26 @@
+name: Find Latest CodeQL Bundle
+description: Finds the URL of the latest released version of the CodeQL bundle.
+outputs:
+  url:
+    description: The download URL of the latest CodeQL bundle release
+    value: ${{ steps.find-latest.outputs.url }}
+runs:
+  using: composite
+  steps:
+    - name: Find Latest Release
+      id: find-latest
+      shell: pwsh
+      run: |
+        $Latest = gh release list --repo github/codeql-action --exclude-drafts --limit 1000 |
+          ForEach-Object { $C = $_ -split "`t"; return @{ type = $C[1]; tag = $C[2]; } } |
+          Where-Object { $_.type -eq 'Latest' }
+
+        $Tag = $Latest.tag
+        if ($Tag -eq '') {
+          throw 'Failed to find latest bundle release.'
+        }
+
+        Write-Output "Latest bundle tag is '${Tag}'."
+        "url=https://github.com/github/codeql-action/releases/download/${Tag}/codeql-bundle-linux64.tar.gz" >> $env:GITHUB_OUTPUT
+      env:
+        GITHUB_TOKEN: ${{ github.token }}

.github/workflows/csharp-qltest.yml

@@ -67,7 +67,7 @@ jobs:
           mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
           # Safe guard against using the bundled extractor
           rm -rf "$CODEQL_PATH/csharp"
-          codeql test run --threads=0 --ram 52000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}
   unit-tests:

.github/workflows/js-ml-tests.yml

@@ -23,9 +23,9 @@ defaults:
     working-directory: javascript/ql/experimental/adaptivethreatmodeling
 
 jobs:
-  qlcompile:
-    name: Check QL compilation
-    runs-on: ubuntu-latest
+  qltest:
+    name: Test QL
+    runs-on: ubuntu-latest-xl
     steps:
       - uses: actions/checkout@v3
 
@@ -33,36 +33,33 @@ jobs:
 
       - name: Install pack dependencies
         run: |
-          for pack in modelbuilding src; do
+          for pack in modelbuilding src test; do
             codeql pack install --mode verify -- "${pack}"
           done
+      
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: js-ml-test
 
       - name: Check QL compilation
         run: |
           codeql query compile \
             --check-only \
-            --ram 5120 \
+            --ram 50000 \
             --additional-packs "${{ github.workspace }}" \
             --threads=0 \
+            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
             -- \
             lib modelbuilding src
 
-  qltest:
-    name: Run QL tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: ./.github/actions/fetch-codeql
-
-      - name: Install pack dependencies
-        run: codeql pack install -- test
-
       - name: Run QL tests
         run: |
           codeql test run \
             --threads=0 \
-            --ram 5120 \
+            --ram 50000 \
             --additional-packs "${{ github.workspace }}" \
+            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
             -- \
-            test
+            test

.github/workflows/mad_modelDiff.yml

@@ -61,8 +61,8 @@ jobs:
             DATABASE=$2
             cd codeql-$QL_VARIANT
             SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/model-generator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $MODELS/${SHORTNAME}.qll
-            mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
+            python java/ql/src/utils/model-generator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE ${SHORTNAME}.temp.model.yml
+            mv java/ql/lib/ext/generated/${SHORTNAME}.temp.model.yml $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.model.yml
             cd ..
           }
 
@@ -85,16 +85,16 @@ jobs:
           set -x
           MODELS=`pwd`/tmp-models
           ls -1 tmp-models/
-          for m in $MODELS/*_main.qll ; do
+          for m in $MODELS/*_main.model.yml ; do
             t="${m/main/"pr"}"
             basename=`basename $m`
-            name="diff_${basename/_main.qll/""}"
+            name="diff_${basename/_main.model.yml/""}"
             (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
           done
       - uses: actions/upload-artifact@v3
         with:
           name: models
-          path: tmp-models/*.qll
+          path: tmp-models/*.model.yml
           retention-days: 20
       - uses: actions/upload-artifact@v3
         with:

.github/workflows/mad_regenerate-models.yml

@@ -53,7 +53,7 @@ jobs:
           java/ql/src/utils/model-generator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
       - name: Stage changes
         run: |
-          find java -name "*.qll" -print0 | xargs -0 git add
+          find java -name "*.model.yml" -print0 | xargs -0 git add
           git status
           git diff --cached > models.patch
       - uses: actions/upload-artifact@v3

.github/workflows/ql-for-ql-build.yml

@@ -22,11 +22,15 @@ jobs:
     steps:
       ### Build the queries ###
       - uses: actions/checkout@v3
+      - name: Find latest bundle
+        id: find-latest-bundle
+        uses: ./.github/actions/find-latest-bundle
       - name: Find codeql
         id: find-codeql
         uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
         with:
           languages: javascript # does not matter
+          tools: ${{ steps.find-latest-bundle.outputs.url }}
       - name: Get CodeQL version
         id: get-codeql-version
         run: |
@@ -138,6 +142,7 @@ jobs:
           languages: ql
           db-location: ${{ runner.temp }}/db
           config-file: ./ql-for-ql-config.yml
+          tools: ${{ steps.find-latest-bundle.outputs.url }}
       - name: Move pack cache
         run: |
           cp -r ${PACK}/.cache ql/ql/src/.cache

.github/workflows/ruby-qltest.yml

@@ -62,6 +62,6 @@ jobs:
           key: ruby-qltest
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 52000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll

@@ -244,4 +244,20 @@ module Consistency {
     not callable = viableCallable(call) and
     not any(ConsistencyConfiguration c).viableImplInCallContextTooLargeExclude(call, ctx, callable)
   }
+
+  query predicate uniqueParameterNodeAtPosition(
+    DataFlowCallable c, ParameterPosition pos, Node p, string msg
+  ) {
+    isParameterNode(p, c, pos) and
+    not exists(unique(Node p0 | isParameterNode(p0, c, pos))) and
+    msg = "Parameters with overlapping positions."
+  }
+
+  query predicate uniqueParameterNodePosition(
+    DataFlowCallable c, ParameterPosition pos, Node p, string msg
+  ) {
+    isParameterNode(p, c, pos) and
+    not exists(unique(ParameterPosition pos0 | isParameterNode(p, c, pos0))) and
+    msg = "Parameter node with multiple positions."
+  }
 }

cpp/ql/lib/semmle/code/cpp/Location.qll

@@ -63,6 +63,7 @@ class Location extends @location {
    * For more information, see
    * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
+  pragma[inline]
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
   ) {

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplConsistency.qll

@@ -244,4 +244,20 @@ module Consistency {
     not callable = viableCallable(call) and
     not any(ConsistencyConfiguration c).viableImplInCallContextTooLargeExclude(call, ctx, callable)
   }
+
+  query predicate uniqueParameterNodeAtPosition(
+    DataFlowCallable c, ParameterPosition pos, Node p, string msg
+  ) {
+    isParameterNode(p, c, pos) and
+    not exists(unique(Node p0 | isParameterNode(p0, c, pos))) and
+    msg = "Parameters with overlapping positions."
+  }
+
+  query predicate uniqueParameterNodePosition(
+    DataFlowCallable c, ParameterPosition pos, Node p, string msg
+  ) {
+    isParameterNode(p, c, pos) and
+    not exists(unique(ParameterPosition pos0 | isParameterNode(p, c, pos0))) and
+    msg = "Parameter node with multiple positions."
+  }
 }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/DefaultTaintTracking.qll

@@ -472,6 +472,7 @@ module TaintedWithPath {
      * For more information, see
      * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
      */
+    pragma[inline]
     predicate hasLocationInfo(
       string filepath, int startline, int startcolumn, int endline, int endcolumn
     ) {

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll

@@ -244,4 +244,20 @@ module Consistency {
     not callable = viableCallable(call) and
     not any(ConsistencyConfiguration c).viableImplInCallContextTooLargeExclude(call, ctx, callable)
   }
+
+  query predicate uniqueParameterNodeAtPosition(
+    DataFlowCallable c, ParameterPosition pos, Node p, string msg
+  ) {
+    isParameterNode(p, c, pos) and
+    not exists(unique(Node p0 | isParameterNode(p0, c, pos))) and
+    msg = "Parameters with overlapping positions."
+  }
+
+  query predicate uniqueParameterNodePosition(
+    DataFlowCallable c, ParameterPosition pos, Node p, string msg
+  ) {
+    isParameterNode(p, c, pos) and
+    not exists(unique(ParameterPosition pos0 | isParameterNode(p, c, pos0))) and
+    msg = "Parameter node with multiple positions."
+  }
 }

cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll

@@ -31,7 +31,17 @@ private class IteratorTraits extends Class {
  * `std::iterator_traits` instantiation for it.
  */
 private class IteratorByTraits extends Iterator {
-  IteratorByTraits() { exists(IteratorTraits it | it.getIteratorType() = this) }
+  IteratorTraits trait;
+
+  IteratorByTraits() { trait.getIteratorType() = this }
+
+  override Type getValueType() {
+    exists(TypedefType t |
+      trait.getAMember() = t and
+      t.getName() = "value_type" and
+      result = t.getUnderlyingType()
+    )
+  }
 }
 
 /**
@@ -42,20 +52,27 @@ private class IteratorByTraits extends Iterator {
  */
 private class IteratorByPointer extends Iterator instanceof PointerType {
   IteratorByPointer() { not this instanceof IteratorByTraits }
+
+  override Type getValueType() { result = super.getBaseType() }
 }
 
 /**
  * A type which has the typedefs expected for an iterator.
  */
 private class IteratorByTypedefs extends Iterator, Class {
+  TypedefType valueType;
+
   IteratorByTypedefs() {
     this.getAMember().(TypedefType).hasName("difference_type") and
-    this.getAMember().(TypedefType).hasName("value_type") and
+    valueType = this.getAMember() and
+    valueType.hasName("value_type") and
     this.getAMember().(TypedefType).hasName("pointer") and
     this.getAMember().(TypedefType).hasName("reference") and
     this.getAMember().(TypedefType).hasName("iterator_category") and
     not this.hasQualifiedName(["std", "bsl"], "iterator_traits")
   }
+
+  override Type getValueType() { result = valueType.getUnderlyingType() }
 }
 
 /**
@@ -63,6 +80,8 @@ private class IteratorByTypedefs extends Iterator, Class {
  */
 private class StdIterator extends Iterator, Class {
   StdIterator() { this.hasQualifiedName(["std", "bsl"], "iterator") }
+
+  override Type getValueType() { result = this.getTemplateArgument(1).(Type).getUnderlyingType() }
 }
 
 /**
@@ -166,12 +185,15 @@ private class IteratorSubOperator extends Operator, TaintFunction {
 /**
  * A non-member `operator+=` or `operator-=` function for an iterator type.
  */
-private class IteratorAssignArithmeticOperator extends Operator, DataFlowFunction, TaintFunction {
+class IteratorAssignArithmeticOperator extends Operator {
   IteratorAssignArithmeticOperator() {
     this.hasName(["operator+=", "operator-="]) and
     exists(getIteratorArgumentInput(this, 0))
   }
+}
 
+private class IteratorAssignArithmeticOperatorModel extends IteratorAssignArithmeticOperator,
+  DataFlowFunction, TaintFunction {
   override predicate hasDataFlow(FunctionInput input, FunctionOutput output) {
     input.isParameter(0) and
     output.isReturnValue()
@@ -210,11 +232,14 @@ class IteratorPointerDereferenceMemberOperator extends MemberFunction, TaintFunc
 /**
  * An `operator++` or `operator--` member function for an iterator type.
  */
-private class IteratorCrementMemberOperator extends MemberFunction, DataFlowFunction, TaintFunction {
+class IteratorCrementMemberOperator extends MemberFunction {
   IteratorCrementMemberOperator() {
     this.getClassAndName(["operator++", "operator--"]) instanceof Iterator
   }
+}
 
+private class IteratorCrementMemberOperatorModel extends IteratorCrementMemberOperator,
+  DataFlowFunction, TaintFunction {
   override predicate hasDataFlow(FunctionInput input, FunctionOutput output) {
     input.isQualifierAddress() and
     output.isReturnValue()

cpp/ql/lib/semmle/code/cpp/models/implementations/StdContainer.qll

@@ -5,38 +5,53 @@
 import semmle.code.cpp.models.interfaces.Taint
 import semmle.code.cpp.models.interfaces.Iterator
 
+/**
+ * A sequence container template class (for example, `std::vector`) from the
+ * standard library.
+ */
+abstract class StdSequenceContainer extends Class {
+  Type getElementType() { result = this.getTemplateArgument(0) }
+}
+
 /**
  * The `std::array` template class.
  */
-private class Array extends Class {
+private class Array extends StdSequenceContainer {
   Array() { this.hasQualifiedName(["std", "bsl"], "array") }
 }
 
+/**
+ * The `std::string` template class.
+ */
+private class String extends StdSequenceContainer {
+  String() { this.hasQualifiedName(["std", "bsl"], "basic_string") }
+}
+
 /**
  * The `std::deque` template class.
  */
-private class Deque extends Class {
+private class Deque extends StdSequenceContainer {
   Deque() { this.hasQualifiedName(["std", "bsl"], "deque") }
 }
 
 /**
  * The `std::forward_list` template class.
  */
-private class ForwardList extends Class {
+private class ForwardList extends StdSequenceContainer {
   ForwardList() { this.hasQualifiedName(["std", "bsl"], "forward_list") }
 }
 
 /**
  * The `std::list` template class.
  */
-private class List extends Class {
+private class List extends StdSequenceContainer {
   List() { this.hasQualifiedName(["std", "bsl"], "list") }
 }
 
 /**
  * The `std::vector` template class.
  */
-private class Vector extends Class {
+private class Vector extends StdSequenceContainer {
   Vector() { this.hasQualifiedName(["std", "bsl"], "vector") }
 }
 

cpp/ql/lib/semmle/code/cpp/models/implementations/StdString.qll

@@ -15,15 +15,6 @@ private class StdBasicString extends ClassTemplateInstantiation {
   StdBasicString() { this.hasQualifiedName(["std", "bsl"], "basic_string") }
 }
 
-/**
- * The `std::basic_string::iterator` declaration.
- */
-private class StdBasicStringIterator extends Iterator, Type {
-  StdBasicStringIterator() {
-    this.getEnclosingElement() instanceof StdBasicString and this.hasName("iterator")
-  }
-}
-
 /**
  * A `std::string` function for which taint should be propagated.
  */

cpp/ql/lib/semmle/code/cpp/models/interfaces/Iterator.qll

@@ -29,5 +29,17 @@ abstract class GetIteratorFunction extends Function {
 
 /**
  * A type which can be used as an iterator.
+ *
+ * Note: Do _not_ `extend` when inheriting from this class in queries. Always use `instanceof`:
+ * ```
+ * class MyIterator instanceof Iterator { ... }
+ * ```
  */
-abstract class Iterator extends Type { }
+abstract class Iterator extends Type {
+  /**
+   * Gets the value type of this iterator, if any.
+   *
+   * For example, the value type of a `std::vector<int>::iterator` is `int`.
+   */
+  Type getValueType() { none() }
+}

cpp/ql/src/Security/CWE/CWE-022/TaintedPath.ql

@@ -17,8 +17,9 @@
 import cpp
 import semmle.code.cpp.security.FunctionWithWrappers
 import semmle.code.cpp.security.Security
-import semmle.code.cpp.security.TaintTracking
-import TaintedWithPath
+import semmle.code.cpp.ir.IR
+import semmle.code.cpp.ir.dataflow.TaintTracking
+import DataFlow::PathGraph
 
 /**
  * A function for opening a file.
@@ -46,18 +47,91 @@ class FileFunction extends FunctionWithWrappers {
   override predicate interestingArg(int arg) { arg = 0 }
 }
 
-class TaintedPathConfiguration extends TaintTrackingConfiguration {
-  override predicate isSink(Element tainted) {
-    exists(FileFunction fileFunction | fileFunction.outermostWrapperFunctionCall(tainted, _))
+Expr asSourceExpr(DataFlow::Node node) {
+  result = node.asConvertedExpr()
+  or
+  result = node.asDefiningArgument()
+}
+
+Expr asSinkExpr(DataFlow::Node node) {
+  result =
+    node.asOperand()
+        .(SideEffectOperand)
+        .getUse()
+        .(ReadSideEffectInstruction)
+        .getArgumentDef()
+        .getUnconvertedResultExpression()
+}
+
+/**
+ * Holds for a variable that has any kind of upper-bound check anywhere in the program.
+ * This is biased towards being inclusive and being a coarse overapproximation because
+ * there are a lot of valid ways of doing an upper bounds checks if we don't consider
+ * where it occurs, for example:
+ * ```cpp
+ *   if (x < 10) { sink(x); }
+ *
+ *   if (10 > y) { sink(y); }
+ *
+ *   if (z > 10) { z = 10; }
+ *   sink(z);
+ * ```
+ */
+predicate hasUpperBoundsCheck(Variable var) {
+  exists(RelationalOperation oper, VariableAccess access |
+    oper.getAnOperand() = access and
+    access.getTarget() = var and
+    // Comparing to 0 is not an upper bound check
+    not oper.getAnOperand().getValue() = "0"
+  )
+}
+
+class TaintedPathConfiguration extends TaintTracking::Configuration {
+  TaintedPathConfiguration() { this = "TaintedPathConfiguration" }
+
+  override predicate isSource(DataFlow::Node node) { isUserInput(asSourceExpr(node), _) }
+
+  override predicate isSink(DataFlow::Node node) {
+    exists(FileFunction fileFunction |
+      fileFunction.outermostWrapperFunctionCall(asSinkExpr(node), _)
+    )
+  }
+
+  override predicate isSanitizerIn(DataFlow::Node node) { this.isSource(node) }
+
+  override predicate isSanitizer(DataFlow::Node node) {
+    node.asExpr().(Call).getTarget().getUnspecifiedType() instanceof ArithmeticType
+    or
+    exists(LoadInstruction load, Variable checkedVar |
+      load = node.asInstruction() and
+      checkedVar = load.getSourceAddress().(VariableAddressInstruction).getAstVariable() and
+      hasUpperBoundsCheck(checkedVar)
+    )
+  }
+
+  predicate hasFilteredFlowPath(DataFlow::PathNode source, DataFlow::PathNode sink) {
+    this.hasFlowPath(source, sink) and
+    // The use of `isUserInput` in `isSink` in combination with `asSourceExpr` causes
+    // duplicate results. Filter these duplicates. The proper solution is to switch to
+    // using `LocalFlowSource` and `RemoteFlowSource`, but this currently only supports
+    // a subset of the cases supported by `isUserInput`.
+    not exists(DataFlow::PathNode source2 |
+      this.hasFlowPath(source2, sink) and
+      asSourceExpr(source.getNode()) = asSourceExpr(source2.getNode())
+    |
+      not exists(source.getNode().asConvertedExpr()) and exists(source2.getNode().asConvertedExpr())
+    )
   }
 }
 
 from
-  FileFunction fileFunction, Expr taintedArg, Expr taintSource, PathNode sourceNode,
-  PathNode sinkNode, string taintCause, string callChain
+  FileFunction fileFunction, Expr taintedArg, Expr taintSource, TaintedPathConfiguration cfg,
+  DataFlow::PathNode sourceNode, DataFlow::PathNode sinkNode, string taintCause, string callChain
 where
+  taintedArg = asSinkExpr(sinkNode.getNode()) and
   fileFunction.outermostWrapperFunctionCall(taintedArg, callChain) and
-  taintedWithPath(taintSource, taintedArg, sourceNode, sinkNode) and
+  cfg.hasFilteredFlowPath(sourceNode, sinkNode) and
+  taintSource = asSourceExpr(sourceNode.getNode()) and
   isUserInput(taintSource, taintCause)
 select taintedArg, sourceNode, sinkNode,
   "This argument to a file access function is derived from $@ and then passed to " + callChain + ".",

cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected

@@ -93,3 +93,5 @@ postWithInFlow
 | test.cpp:499:4:499:4 | p [inner post update] | PostUpdateNode should not be the target of local flow. |
 | test.cpp:505:35:505:35 | x [inner post update] | PostUpdateNode should not be the target of local flow. |
 viableImplInCallContextTooLarge
+uniqueParameterNodeAtPosition
+uniqueParameterNodePosition

cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-ir-consistency.expected

@@ -637,3 +637,5 @@ postWithInFlow
 | true_upon_entry.cpp:101:18:101:18 | VariableAddress [post update] | PostUpdateNode should not be the target of local flow. |
 | true_upon_entry.cpp:102:5:102:5 | x [post update] | PostUpdateNode should not be the target of local flow. |
 viableImplInCallContextTooLarge
+uniqueParameterNodeAtPosition
+uniqueParameterNodePosition

cpp/ql/test/library-tests/dataflow/fields/dataflow-consistency.expected

@@ -158,3 +158,5 @@ postWithInFlow
 | struct_init.c:24:11:24:12 | ab [inner post update] | PostUpdateNode should not be the target of local flow. |
 | struct_init.c:36:17:36:24 | nestedAB [inner post update] | PostUpdateNode should not be the target of local flow. |
 viableImplInCallContextTooLarge
+uniqueParameterNodeAtPosition
+uniqueParameterNodePosition

cpp/ql/test/library-tests/dataflow/fields/dataflow-ir-consistency.expected

@@ -1326,3 +1326,5 @@ postWithInFlow
 | struct_init.c:46:16:46:24 | pointerAB [post update] | PostUpdateNode should not be the target of local flow. |
 | struct_init.c:46:16:46:24 | pointerAB [post update] | PostUpdateNode should not be the target of local flow. |
 viableImplInCallContextTooLarge
+uniqueParameterNodeAtPosition
+uniqueParameterNodePosition

cpp/ql/test/library-tests/syntax-zoo/dataflow-consistency.expected

@@ -127,3 +127,5 @@ postWithInFlow
 | static_init_templates.cpp:21:2:21:4 | val [post update] | PostUpdateNode should not be the target of local flow. |
 | try_catch.cpp:7:8:7:8 | call to exception | PostUpdateNode should not be the target of local flow. |
 viableImplInCallContextTooLarge
+uniqueParameterNodeAtPosition
+uniqueParameterNodePosition

cpp/ql/test/library-tests/syntax-zoo/dataflow-ir-consistency.expected

@@ -2713,3 +2713,7 @@ postWithInFlow
 | whilestmt.c:40:7:40:7 | VariableAddress [post update] | PostUpdateNode should not be the target of local flow. |
 | whilestmt.c:42:7:42:7 | VariableAddress [post update] | PostUpdateNode should not be the target of local flow. |
 viableImplInCallContextTooLarge
+uniqueParameterNodeAtPosition
+| ir.cpp:724:6:724:13 | TryCatch | 0 | ir.cpp:735:22:735:22 | *s | Parameters with overlapping positions. |
+| ir.cpp:724:6:724:13 | TryCatch | 0 | ir.cpp:738:24:738:24 | *e | Parameters with overlapping positions. |
+uniqueParameterNodePosition

cpp/ql/test/query-tests/Security/CWE/CWE-022/SAMATE/TaintedPath/TaintedPath.expected

@@ -1,19 +1,8 @@
 edges
-| CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:55:27:55:38 | ... + ... | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | (const char *)... |
-| CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:55:27:55:38 | ... + ... | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | data |
-| CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:55:27:55:38 | ... + ... | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | data |
-| CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:55:27:55:38 | ... + ... | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | data indirection |
-| CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:55:27:55:38 | fgets output argument | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | (const char *)... |
-| CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:55:27:55:38 | fgets output argument | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | data |
-| CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:55:27:55:38 | fgets output argument | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | data |
 | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:55:27:55:38 | fgets output argument | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | data indirection |
-subpaths
 nodes
-| CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:55:27:55:38 | ... + ... | semmle.label | ... + ... |
 | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:55:27:55:38 | fgets output argument | semmle.label | fgets output argument |
-| CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | (const char *)... | semmle.label | (const char *)... |
-| CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | data | semmle.label | data |
-| CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | data | semmle.label | data |
 | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | data indirection | semmle.label | data indirection |
+subpaths
 #select
-| CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | data | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:55:27:55:38 | ... + ... | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | data | This argument to a file access function is derived from $@ and then passed to fopen(filename). | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:55:27:55:38 | ... + ... | user input (fgets) |
+| CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | data | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:55:27:55:38 | fgets output argument | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:77:23:77:26 | data indirection | This argument to a file access function is derived from $@ and then passed to fopen(filename). | CWE23_Relative_Path_Traversal__char_console_fopen_11.cpp:55:27:55:38 | ... + ... | user input (fgets) |

cpp/ql/test/query-tests/Security/CWE/CWE-022/semmle/tests/TaintedPath.expected

@@ -1,19 +1,24 @@
 edges
-| test.c:9:23:9:26 | argv | test.c:17:11:17:18 | (const char *)... |
-| test.c:9:23:9:26 | argv | test.c:17:11:17:18 | (const char *)... |
-| test.c:9:23:9:26 | argv | test.c:17:11:17:18 | fileName |
-| test.c:9:23:9:26 | argv | test.c:17:11:17:18 | fileName |
-| test.c:9:23:9:26 | argv | test.c:17:11:17:18 | fileName |
-| test.c:9:23:9:26 | argv | test.c:17:11:17:18 | fileName |
 | test.c:9:23:9:26 | argv | test.c:17:11:17:18 | fileName indirection |
-| test.c:9:23:9:26 | argv | test.c:17:11:17:18 | fileName indirection |
-subpaths
+| test.c:31:22:31:25 | argv | test.c:32:11:32:18 | fileName indirection |
+| test.c:37:17:37:24 | fileName | test.c:38:11:38:18 | fileName indirection |
+| test.c:37:17:37:24 | scanf output argument | test.c:38:11:38:18 | fileName indirection |
+| test.c:43:17:43:24 | fileName | test.c:44:11:44:18 | fileName indirection |
+| test.c:43:17:43:24 | scanf output argument | test.c:44:11:44:18 | fileName indirection |
 nodes
 | test.c:9:23:9:26 | argv | semmle.label | argv |
-| test.c:9:23:9:26 | argv | semmle.label | argv |
-| test.c:17:11:17:18 | (const char *)... | semmle.label | (const char *)... |
-| test.c:17:11:17:18 | fileName | semmle.label | fileName |
-| test.c:17:11:17:18 | fileName | semmle.label | fileName |
 | test.c:17:11:17:18 | fileName indirection | semmle.label | fileName indirection |
+| test.c:31:22:31:25 | argv | semmle.label | argv |
+| test.c:32:11:32:18 | fileName indirection | semmle.label | fileName indirection |
+| test.c:37:17:37:24 | fileName | semmle.label | fileName |
+| test.c:37:17:37:24 | scanf output argument | semmle.label | scanf output argument |
+| test.c:38:11:38:18 | fileName indirection | semmle.label | fileName indirection |
+| test.c:43:17:43:24 | fileName | semmle.label | fileName |
+| test.c:43:17:43:24 | scanf output argument | semmle.label | scanf output argument |
+| test.c:44:11:44:18 | fileName indirection | semmle.label | fileName indirection |
+subpaths
 #select
-| test.c:17:11:17:18 | fileName | test.c:9:23:9:26 | argv | test.c:17:11:17:18 | fileName | This argument to a file access function is derived from $@ and then passed to fopen(filename). | test.c:9:23:9:26 | argv | user input (argv) |
+| test.c:17:11:17:18 | fileName | test.c:9:23:9:26 | argv | test.c:17:11:17:18 | fileName indirection | This argument to a file access function is derived from $@ and then passed to fopen(filename). | test.c:9:23:9:26 | argv | user input (argv) |
+| test.c:32:11:32:18 | fileName | test.c:31:22:31:25 | argv | test.c:32:11:32:18 | fileName indirection | This argument to a file access function is derived from $@ and then passed to fopen(filename). | test.c:31:22:31:25 | argv | user input (argv) |
+| test.c:38:11:38:18 | fileName | test.c:37:17:37:24 | fileName | test.c:38:11:38:18 | fileName indirection | This argument to a file access function is derived from $@ and then passed to fopen(filename). | test.c:37:17:37:24 | fileName | user input (scanf) |
+| test.c:44:11:44:18 | fileName | test.c:43:17:43:24 | fileName | test.c:44:11:44:18 | fileName indirection | This argument to a file access function is derived from $@ and then passed to fopen(filename). | test.c:43:17:43:24 | fileName | user input (scanf) |

cpp/ql/test/query-tests/Security/CWE/CWE-022/semmle/tests/stdlib.h

@@ -11,3 +11,7 @@ FILE *fopen(const char *filename, const char *mode);
 int sprintf(char *s, const char *format, ...);
 size_t strlen(const char *s);
 char *strncat(char *s1, const char *s2, size_t n);
+int scanf(const char *format, ...);
+void *malloc(size_t size);
+double strtod(const char *ptr, char **endptr);
+char *getenv(const char *name);

cpp/ql/test/query-tests/Security/CWE/CWE-022/semmle/tests/test.c

@@ -26,5 +26,38 @@ int main(int argc, char** argv) {
     strncat(fileName+len, fixed, FILENAME_MAX-len-1);
     fopen(fileName, "wb+");
   }
+
+  {
+    char *fileName = argv[1];
+    fopen(fileName, "wb+"); // BAD
+  }
+
+  {
+    char fileName[20];
+    scanf("%s", fileName);
+    fopen(fileName, "wb+"); // BAD
+  }
+
+  {
+    char *fileName = (char*)malloc(20 * sizeof(char));
+    scanf("%s", fileName);
+    fopen(fileName, "wb+"); // BAD
+  }
+
+  {
+    char *aNumber = getenv("A_NUMBER");
+    double number = strtod(aNumber, 0);
+    char fileName[20];
+    sprintf(fileName, "/foo/%f", number);
+    fopen(fileName, "wb+"); // GOOD
+  }
+
+  {
+    void read(const char *fileName);
+    read(argv[1]); // BAD [NOT DETECTED]
+  }
 }
 
+void read(char *fileName) {
+  fopen(fileName, "wb+");
+}

csharp/ql/lib/ext/generated/dotnet_runtime.model.yml

@@ -1,6 +1,5 @@
- 
 # THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.
-# Definitions of taint steps in the dotnet_runtime framework.
+# Definitions of models for the dotnet_runtime framework.
 
 extensions:
 

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplConsistency.qll

@@ -244,4 +244,20 @@ module Consistency {
     not callable = viableCallable(call) and
     not any(ConsistencyConfiguration c).viableImplInCallContextTooLargeExclude(call, ctx, callable)
   }
+
+  query predicate uniqueParameterNodeAtPosition(
+    DataFlowCallable c, ParameterPosition pos, Node p, string msg
+  ) {
+    isParameterNode(p, c, pos) and
+    not exists(unique(Node p0 | isParameterNode(p0, c, pos))) and
+    msg = "Parameters with overlapping positions."
+  }
+
+  query predicate uniqueParameterNodePosition(
+    DataFlowCallable c, ParameterPosition pos, Node p, string msg
+  ) {
+    isParameterNode(p, c, pos) and
+    not exists(unique(ParameterPosition pos0 | isParameterNode(p, c, pos0))) and
+    msg = "Parameter node with multiple positions."
+  }
 }

csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll

@@ -520,7 +520,8 @@ module Private {
   predicate summaryParameterNodeRange(SummarizedCallable c, ParameterPosition pos) {
     parameterReadState(c, _, pos)
     or
-    isParameterPostUpdate(_, c, pos)
+    // Same as `isParameterPostUpdate(_, c, pos)`, but can be used in a negative context
+    any(SummaryNodeState state).isOutputState(c, SummaryComponentStack::argument(pos))
   }
 
   private predicate callbackOutput(

csharp/ql/src/Telemetry/SupportedExternalApis.ql

@@ -0,0 +1,24 @@
+/**
+ * @name Usage of supported APIs coming from external libraries
+ * @description A list of supported 3rd party APIs used in the codebase. Excludes APIs exposed by test libraries.
+ * @kind metric
+ * @tags summary telemetry
+ * @id csharp/telemetry/supported-external-api
+ */
+
+private import csharp
+private import semmle.code.csharp.dispatch.Dispatch
+private import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
+private import ExternalApi
+
+private predicate relevant(ExternalApi api) {
+  not api.isUninteresting() and
+  (
+    api.isSupported() or
+    api instanceof FlowSummaryImpl::Public::NegativeSummarizedCallable
+  )
+}
+
+from string info, int usages
+where Results<relevant/1>::restrict(info, usages)
+select info, usages order by usages desc

csharp/ql/src/change-notes/2022-12-01-supported-external-apis-query.md

@@ -0,0 +1,4 @@
+---
+category: newQuery
+---
+* Added a new query, `csharp/telemetry/supported-external-api`, to detect supported 3rd party APIs used in a codebase.

csharp/ql/src/utils/model-generator/GenerateFlowModelExtensions.py

@@ -1,15 +0,0 @@
-#!/usr/bin/python3
-
-import sys
-import os.path
-import subprocess
-
-# Add Model as Data script directory to sys.path.
-gitroot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
-madpath = os.path.join(gitroot, "misc/scripts/models-as-data/")
-sys.path.append(madpath)
-
-import generate_flow_model_extensions as model
-
-language = "csharp"
-model.Generator.make(language).run()

csharp/ql/src/utils/modelconverter/ExtractNegativeSummaries.ql

@@ -0,0 +1,14 @@
+/**
+ * @name Extract MaD negative summary model rows.
+ * @description This extracts the Models as data negative summary model rows.
+ * @id csharp/utils/modelconverter/generate-data-extensions-negative-summary
+ */
+
+import csharp
+import semmle.code.csharp.dataflow.ExternalFlow
+
+from string package, string type, string name, string signature, string provenance
+where
+  negativeSummaryModel(package, type, name, signature, provenance) and
+  provenance != "generated"
+select package, type, name, signature, provenance order by package, type, name, signature

csharp/ql/test/query-tests/Telemetry/SupportedExternalApis/SupportedExternalApis.cs

@@ -0,0 +1,48 @@
+using System;
+using System.Collections.Generic;
+using System.Web;
+
+public class SupportedExternalApis
+{
+    public void M1()
+    {
+        var l = new List<object>(); // Uninteresting parameterless constructor
+        var o = new object(); // Uninteresting parameterless constructor
+        l.Add(o); // Has flow summary
+        l.Add(o); // Has flow summary
+    }
+
+    public void M2()
+    {
+        var d0 = new DateTime(); // Uninteresting parameterless constructor
+        var next0 = d0.AddYears(30); // Has no flow summary, supported as negative summary
+
+        var d1 = new DateTime(2000, 1, 1); // Interesting constructor, supported as negative summary
+        var next1 = next0.AddDays(3); // Has no flow summary, supported as negative summary
+        var next2 = next1.AddYears(5); // Has no flow summary, supported as negative summary
+    }
+
+    public void M3()
+    {
+        var guid1 = Guid.Parse("{12345678-1234-1234-1234-123456789012}"); // Has no flow summary, supported as negative summary
+    }
+
+     public void M4()
+    {
+        var o = new object(); // Uninteresting parameterless constructor
+        var response = new HttpResponse(); // Uninteresting parameterless constructor
+        response.AddHeader("header", "value"); // Unsupported
+        response.AppendHeader("header", "value"); // Unsupported
+        response.Write(o); // Known sink
+        response.WriteFile("filename"); // Known sink
+        response.Write(o); // Known sink
+    }
+
+     public void M5()
+    {
+        var l1 = Console.ReadLine(); // Known source
+        var l2 = Console.ReadLine(); // Known source
+        Console.SetError(Console.Out); // Has no flow summary, supported as negative summary
+        var x = Console.Read(); // Known source
+    }
+}

csharp/ql/test/query-tests/Telemetry/SupportedExternalApis/SupportedExternalApis.expected

@@ -0,0 +1,11 @@
+| System#Console.ReadLine() | 2 |
+| System#DateTime.AddYears(System.Int32) | 2 |
+| System.Collections.Generic#List<>.Add(T) | 2 |
+| System.Web#HttpResponse.Write(System.Object) | 2 |
+| System#Console.Read() | 1 |
+| System#Console.SetError(System.IO.TextWriter) | 1 |
+| System#Console.get_Out() | 1 |
+| System#DateTime.AddDays(System.Double) | 1 |
+| System#DateTime.DateTime(System.Int32,System.Int32,System.Int32) | 1 |
+| System#Guid.Parse(System.String) | 1 |
+| System.Web#HttpResponse.WriteFile(System.String) | 1 |

csharp/ql/test/query-tests/Telemetry/SupportedExternalApis/SupportedExternalApis.qlref

@@ -0,0 +1 @@
+Telemetry/SupportedExternalApis.ql

csharp/ql/test/query-tests/Telemetry/SupportedExternalApis/options

@@ -0,0 +1,2 @@
+semmle-extractor-options: /r:System.Collections.Specialized.dll
+semmle-extractor-options: ${testdir}/../../../resources/stubs/System.Web.cs

go/extractor/extractor.go

@@ -103,29 +103,29 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 	extractUniverseScope()
 	log.Println("Done extracting universe scope.")
 
-	// a map of package path to package root directory (currently the module root or the source directory)
-	pkgRoots := make(map[string]string)
-	// a map of package path to source code directory
-	pkgDirs := make(map[string]string)
+	// a map of package path to source directory and module root directory
+	pkgInfos := make(map[string]util.PkgInfo)
 	// root directories of packages that we want to extract
 	wantedRoots := make(map[string]bool)
 
+	if os.Getenv("CODEQL_EXTRACTOR_GO_FAST_PACKAGE_INFO") != "" {
+		log.Printf("Running go list to resolve package and module directories.")
+		// get all packages information
+		pkgInfos, err = util.GetPkgsInfo(patterns, true, modFlags...)
+		if err != nil {
+			log.Fatalf("Error getting dependency package or module directories: %v.", err)
+		}
+		log.Printf("Done running go list deps: resolved %d packages.", len(pkgInfos))
+	}
+
 	// Do a post-order traversal and extract the package scope of each package
 	packages.Visit(pkgs, func(pkg *packages.Package) bool {
 		return true
 	}, func(pkg *packages.Package) {
 		log.Printf("Processing package %s.", pkg.PkgPath)
 
-		if _, ok := pkgRoots[pkg.PkgPath]; !ok {
-			mdir := util.GetModDir(pkg.PkgPath, modFlags...)
-			pdir := util.GetPkgDir(pkg.PkgPath, modFlags...)
-			// GetModDir returns the empty string if the module directory cannot be determined, e.g. if the package
-			// is not using modules. If this is the case, fall back to the package directory
-			if mdir == "" {
-				mdir = pdir
-			}
-			pkgRoots[pkg.PkgPath] = mdir
-			pkgDirs[pkg.PkgPath] = pdir
+		if _, ok := pkgInfos[pkg.PkgPath]; !ok {
+			pkgInfos[pkg.PkgPath] = util.GetPkgInfo(pkg.PkgPath, modFlags...)
 		}
 
 		log.Printf("Extracting types for package %s.", pkg.PkgPath)
@@ -152,11 +152,14 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 	})
 
 	for _, pkg := range pkgs {
-		if pkgRoots[pkg.PkgPath] == "" {
+		pkgInfo, ok := pkgInfos[pkg.PkgPath]
+		if !ok || pkgInfo.PkgDir == "" {
 			log.Fatalf("Unable to get a source directory for input package %s.", pkg.PkgPath)
 		}
-		wantedRoots[pkgRoots[pkg.PkgPath]] = true
-		wantedRoots[pkgDirs[pkg.PkgPath]] = true
+		wantedRoots[pkgInfo.PkgDir] = true
+		if pkgInfo.ModDir != "" {
+			wantedRoots[pkgInfo.ModDir] = true
+		}
 	}
 
 	log.Println("Done processing dependencies.")
@@ -174,7 +177,8 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 		return true
 	}, func(pkg *packages.Package) {
 		for root, _ := range wantedRoots {
-			relDir, err := filepath.Rel(root, pkgDirs[pkg.PkgPath])
+			pkgInfo := pkgInfos[pkg.PkgPath]
+			relDir, err := filepath.Rel(root, pkgInfo.PkgDir)
 			if err != nil || noExtractRe.MatchString(relDir) {
 				// if the path can't be made relative or matches the noExtract regexp skip it
 				continue
@@ -182,8 +186,12 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 
 			extraction.extractPackage(pkg)
 
-			if pkgRoots[pkg.PkgPath] != "" {
-				modPath := filepath.Join(pkgRoots[pkg.PkgPath], "go.mod")
+			modDir := pkgInfo.ModDir
+			if modDir == "" {
+				modDir = pkgInfo.PkgDir
+			}
+			if modDir != "" {
+				modPath := filepath.Join(modDir, "go.mod")
 				if util.FileExists(modPath) {
 					log.Printf("Extracting %s", modPath)
 					start := time.Now()

go/extractor/util/util.go

@@ -1,7 +1,9 @@
 package util
 
 import (
+	"encoding/json"
 	"errors"
+	"io"
 	"log"
 	"os"
 	"os/exec"
@@ -31,13 +33,13 @@ func Getenv(key string, aliases ...string) string {
 
 // runGoList is a helper function for running go list with format `format` and flags `flags` on
 // package `pkgpath`.
-func runGoList(format string, pkgpath string, flags ...string) (string, error) {
-	return runGoListWithEnv(format, pkgpath, nil, flags...)
+func runGoList(format string, patterns []string, flags ...string) (string, error) {
+	return runGoListWithEnv(format, patterns, nil, flags...)
 }
 
-func runGoListWithEnv(format string, pkgpath string, additionalEnv []string, flags ...string) (string, error) {
+func runGoListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) (string, error) {
 	args := append([]string{"list", "-e", "-f", format}, flags...)
-	args = append(args, pkgpath)
+	args = append(args, patterns...)
 	cmd := exec.Command("go", args...)
 	cmd.Env = append(os.Environ(), additionalEnv...)
 	out, err := cmd.Output()
@@ -54,18 +56,89 @@ func runGoListWithEnv(format string, pkgpath string, additionalEnv []string, fla
 	return strings.TrimSpace(string(out)), nil
 }
 
+// PkgInfo holds package directory and module directory (if any) for a package
+type PkgInfo struct {
+	PkgDir string // the directory directly containing source code of this package
+	ModDir string // the module directory containing this package, empty if not a module
+}
+
+// GetPkgsInfo gets the absolute module and package root directories for the packages matched by the
+// patterns `patterns`. It passes to `go list` the flags specified by `flags`.  If `includingDeps`
+// is true, all dependencies will also be included.
+func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[string]PkgInfo, error) {
+	// enable module mode so that we can find a module root if it exists, even if go module support is
+	// disabled by a build
+	if includingDeps {
+		// the flag `-deps` causes all dependencies to be retrieved
+		flags = append(flags, "-deps")
+	}
+
+	// using -json overrides -f format
+	output, err := runGoList("", patterns, append(flags, "-json")...)
+	if err != nil {
+		return nil, err
+	}
+
+	// the output of `go list -json` is a stream of json object
+	type goListPkgInfo struct {
+		ImportPath string
+		Dir        string
+		Module     *struct {
+			Dir string
+		}
+	}
+	pkgInfoMapping := make(map[string]PkgInfo)
+	streamDecoder := json.NewDecoder(strings.NewReader(output))
+	for {
+		var pkgInfo goListPkgInfo
+		decErr := streamDecoder.Decode(&pkgInfo)
+		if decErr == io.EOF {
+			break
+		}
+		if decErr != nil {
+			log.Printf("Error decoding output of go list -json: %s", err.Error())
+			return nil, decErr
+		}
+		pkgAbsDir, err := filepath.Abs(pkgInfo.Dir)
+		if err != nil {
+			log.Printf("Unable to make package dir %s absolute: %s", pkgInfo.Dir, err.Error())
+		}
+		var modAbsDir string
+		if pkgInfo.Module != nil {
+			modAbsDir, err = filepath.Abs(pkgInfo.Module.Dir)
+			if err != nil {
+				log.Printf("Unable to make module dir %s absolute: %s", pkgInfo.Module.Dir, err.Error())
+			}
+		}
+		pkgInfoMapping[pkgInfo.ImportPath] = PkgInfo{
+			PkgDir: pkgAbsDir,
+			ModDir: modAbsDir,
+		}
+	}
+	return pkgInfoMapping, nil
+}
+
+// GetPkgInfo fills the package info structure for the specified package path.
+// It passes the `go list` the flags specified by `flags`.
+func GetPkgInfo(pkgpath string, flags ...string) PkgInfo {
+	return PkgInfo{
+		PkgDir: GetPkgDir(pkgpath, flags...),
+		ModDir: GetModDir(pkgpath, flags...),
+	}
+}
+
 // GetModDir gets the absolute directory of the module containing the package with path
 // `pkgpath`. It passes the `go list` the flags specified by `flags`.
 func GetModDir(pkgpath string, flags ...string) string {
 	// enable module mode so that we can find a module root if it exists, even if go module support is
 	// disabled by a build
-	mod, err := runGoListWithEnv("{{.Module}}", pkgpath, []string{"GO111MODULE=on"}, flags...)
+	mod, err := runGoListWithEnv("{{.Module}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...)
 	if err != nil || mod == "<nil>" {
 		// if the command errors or modules aren't being used, return the empty string
 		return ""
 	}
 
-	modDir, err := runGoListWithEnv("{{.Module.Dir}}", pkgpath, []string{"GO111MODULE=on"}, flags...)
+	modDir, err := runGoListWithEnv("{{.Module.Dir}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...)
 	if err != nil {
 		return ""
 	}
@@ -81,7 +154,7 @@ func GetModDir(pkgpath string, flags ...string) string {
 // GetPkgDir gets the absolute directory containing the package with path `pkgpath`. It passes the
 // `go list` command the flags specified by `flags`.
 func GetPkgDir(pkgpath string, flags ...string) string {
-	pkgDir, err := runGoList("{{.Dir}}", pkgpath, flags...)
+	pkgDir, err := runGoList("{{.Dir}}", []string{pkgpath}, flags...)
 	if err != nil {
 		return ""
 	}
@@ -97,7 +170,7 @@ func GetPkgDir(pkgpath string, flags ...string) string {
 // DepErrors checks there are any errors resolving dependencies for `pkgpath`. It passes the `go
 // list` command the flags specified by `flags`.
 func DepErrors(pkgpath string, flags ...string) bool {
-	out, err := runGoList("{{if .DepsErrors}}{{else}}error{{end}}", pkgpath, flags...)
+	out, err := runGoList("{{if .DepsErrors}}{{else}}error{{end}}", []string{pkgpath}, flags...)
 	if err != nil {
 		// if go list failed, assume dependencies are broken
 		return false

go/ql/lib/change-notes/2022-12-03-gorqlite-goframe.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Queries that care about SQL, such as `go/sql-injection`, now recognise SQL-consuming functions belonging to the `gorqlite` and `GoFrame` packages.

go/ql/lib/semmle/go/Scopes.qll

@@ -95,7 +95,12 @@ class Entity extends @object {
   /** Gets the package in which this entity is declared, if any. */
   Package getPackage() { result.getScope() = this.getScope() }
 
-  /** Holds if this entity is declared in a package with path `pkg` and has the given `name`. */
+  /**
+   * Holds if this entity is declared in a package with path `pkg` and has the given `name`.
+   *
+   * Note that for methods `pkg` is the package path followed by `.` followed
+   * by the name of the receiver type, for example `io.Writer`.
+   */
   predicate hasQualifiedName(string pkg, string name) {
     pkg = this.getPackage().getPath() and
     name = this.getName()

go/ql/lib/semmle/go/dataflow/internal/FlowSummaryImpl.qll

@@ -520,7 +520,8 @@ module Private {
   predicate summaryParameterNodeRange(SummarizedCallable c, ParameterPosition pos) {
     parameterReadState(c, _, pos)
     or
-    isParameterPostUpdate(_, c, pos)
+    // Same as `isParameterPostUpdate(_, c, pos)`, but can be used in a negative context
+    any(SummaryNodeState state).isOutputState(c, SummaryComponentStack::argument(pos))
   }
 
   private predicate callbackOutput(

go/ql/lib/semmle/go/frameworks/SQL.qll

@@ -103,6 +103,14 @@ module SQL {
     /** A string that might identify package `go-pg/pg/orm` or a specific version of it. */
     private string gopgorm() { result = package("github.com/go-pg/pg", "orm") }
 
+    /** A string that might identify package `github.com/rqlite/gorqlite` or `github.com/raindog308/gorqlite` or a specific version of it. */
+    private string gorqlite() {
+      result = package(["github.com/rqlite/gorqlite", "github.com/raindog308/gorqlite"], "")
+    }
+
+    /** A string that might identify package `github.com/gogf/gf/database/gdb` or a specific version of it. */
+    private string gogf() { result = package("github.com/gogf/gf", "database/gdb") }
+
     /**
      * A string argument to an API of `go-pg/pg` that is directly interpreted as SQL without
      * taking syntactic structure into account.
@@ -152,6 +160,65 @@ module SQL {
       }
     }
 
+    /**
+     * A string argument to an API of `github.com/rqlite/gorqlite`, or a specific version of it, that is directly interpreted as SQL without
+     * taking syntactic structure into account.
+     */
+    private class GorqliteQueryString extends Range {
+      GorqliteQueryString() {
+        // func (conn *Connection) Query(sqlStatements []string) (results []QueryResult, err error)
+        // func (conn *Connection) QueryOne(sqlStatement string) (qr QueryResult, err error)
+        // func (conn *Connection) Queue(sqlStatements []string) (seq int64, err error)
+        // func (conn *Connection) QueueOne(sqlStatement string) (seq int64, err error)
+        // func (conn *Connection) Write(sqlStatements []string) (results []WriteResult, err error)
+        // func (conn *Connection) WriteOne(sqlStatement string) (wr WriteResult, err error)
+        exists(Method m, string name | m.hasQualifiedName(gorqlite(), "Connection", name) |
+          name = ["Query", "QueryOne", "Queue", "QueueOne", "Write", "WriteOne"] and
+          this = m.getACall().getArgument(0)
+        )
+      }
+    }
+
+    /**
+     * A string argument to an API of `github.com/gogf/gf/database/gdb`, or a specific version of it, that is directly interpreted as SQL without
+     * taking syntactic structure into account.
+     */
+    private class GogfQueryString extends Range {
+      GogfQueryString() {
+        exists(Method m, string name | m.implements(gogf(), ["DB", "Core", "TX"], name) |
+          // func (c *Core) Exec(sql string, args ...interface{}) (result sql.Result, err error)
+          // func (c *Core) GetAll(sql string, args ...interface{}) (Result, error)
+          // func (c *Core) GetArray(sql string, args ...interface{}) ([]Value, error)
+          // func (c *Core) GetCount(sql string, args ...interface{}) (int, error)
+          // func (c *Core) GetOne(sql string, args ...interface{}) (Record, error)
+          // func (c *Core) GetValue(sql string, args ...interface{}) (Value, error)
+          // func (c *Core) Prepare(sql string, execOnMaster ...bool) (*Stmt, error)
+          // func (c *Core) Query(sql string, args ...interface{}) (rows *sql.Rows, err error)
+          // func (c *Core) Raw(rawSql string, args ...interface{}) *Model
+          name =
+            [
+              "Query", "Exec", "Prepare", "GetAll", "GetOne", "GetValue", "GetArray", "GetCount",
+              "Raw"
+            ] and
+          this = m.getACall().getArgument(0)
+          or
+          // func (c *Core) GetScan(pointer interface{}, sql string, args ...interface{}) error
+          // func (c *Core) GetStruct(pointer interface{}, sql string, args ...interface{}) error
+          // func (c *Core) GetStructs(pointer interface{}, sql string, args ...interface{}) error
+          name = ["GetScan", "GetStruct", "GetStructs"] and
+          this = m.getACall().getArgument(1)
+          or
+          // func (c *Core) DoCommit(ctx context.Context, link Link, sql string, args []interface{}) (newSql string, newArgs []interface{}, err error)
+          // func (c *Core) DoExec(ctx context.Context, link Link, sql string, args ...interface{}) (result sql.Result, err error)
+          // func (c *Core) DoGetAll(ctx context.Context, link Link, sql string, args ...interface{}) (result Result, err error)
+          // func (c *Core) DoPrepare(ctx context.Context, link Link, sql string) (*Stmt, error)
+          // func (c *Core) DoQuery(ctx context.Context, link Link, sql string, args ...interface{}) (rows *sql.Rows, err error)
+          name = ["DoGetAll", "DoQuery", "DoExec", "DoCommit", "DoPrepare"] and
+          this = m.getACall().getArgument(2)
+        )
+      }
+    }
+
     /** A taint model for various methods on the struct `Formatter` of `go-pg/pg/orm`. */
     private class PgOrmFormatterFunction extends TaintTracking::FunctionModel, Method {
       FunctionInput i;

go/ql/test/library-tests/semmle/go/frameworks/SQL/gogf/go.mod

@@ -0,0 +1,26 @@
+module main
+
+go 1.19
+
+require github.com/gogf/gf v1.16.9
+
+require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/clbanning/mxj v1.8.5-0.20200714211355-ff02cfb8ea28 // indirect
+	github.com/fatih/color v1.12.0 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
+	github.com/gomodule/redigo v1.8.5 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/grokify/html-strip-tags-go v0.0.1 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	go.opentelemetry.io/otel v1.0.0 // indirect
+	go.opentelemetry.io/otel/trace v1.0.0 // indirect
+	golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect
+	golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect
+	golang.org/x/text v0.3.6 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)

go/ql/test/library-tests/semmle/go/frameworks/SQL/gogf/gogf.expected

@@ -0,0 +1,47 @@
+| gogf.go:12:9:12:11 | sql |
+| gogf.go:13:11:13:13 | sql |
+| gogf.go:14:13:14:15 | sql |
+| gogf.go:15:13:15:15 | sql |
+| gogf.go:16:11:16:13 | sql |
+| gogf.go:17:13:17:15 | sql |
+| gogf.go:18:12:18:14 | sql |
+| gogf.go:19:10:19:12 | sql |
+| gogf.go:20:8:20:10 | sql |
+| gogf.go:21:17:21:19 | sql |
+| gogf.go:22:19:22:21 | sql |
+| gogf.go:23:20:23:22 | sql |
+| gogf.go:24:23:24:25 | sql |
+| gogf.go:25:21:25:23 | sql |
+| gogf.go:26:23:26:25 | sql |
+| gogf.go:27:22:27:24 | sql |
+| gogf.go:28:24:28:26 | sql |
+| gogf.go:32:9:32:11 | sql |
+| gogf.go:33:11:33:13 | sql |
+| gogf.go:34:13:34:15 | sql |
+| gogf.go:35:13:35:15 | sql |
+| gogf.go:36:11:36:13 | sql |
+| gogf.go:37:13:37:15 | sql |
+| gogf.go:38:12:38:14 | sql |
+| gogf.go:39:10:39:12 | sql |
+| gogf.go:40:8:40:10 | sql |
+| gogf.go:41:17:41:19 | sql |
+| gogf.go:42:23:42:25 | sql |
+| gogf.go:43:21:43:23 | sql |
+| gogf.go:44:23:44:25 | sql |
+| gogf.go:45:22:45:24 | sql |
+| gogf.go:46:24:46:26 | sql |
+| gogf.go:51:9:51:11 | sql |
+| gogf.go:52:11:52:13 | sql |
+| gogf.go:53:13:53:15 | sql |
+| gogf.go:54:13:54:15 | sql |
+| gogf.go:55:11:55:13 | sql |
+| gogf.go:56:13:56:15 | sql |
+| gogf.go:57:12:57:14 | sql |
+| gogf.go:58:10:58:12 | sql |
+| gogf.go:59:8:59:10 | sql |
+| gogf.go:60:17:60:19 | sql |
+| gogf.go:61:23:61:25 | sql |
+| gogf.go:62:21:62:23 | sql |
+| gogf.go:63:23:63:25 | sql |
+| gogf.go:64:22:64:24 | sql |
+| gogf.go:65:24:65:26 | sql |

go/ql/test/library-tests/semmle/go/frameworks/SQL/gogf/gogf.go

@@ -0,0 +1,70 @@
+package main
+
+//go:generate depstubber -vendor github.com/gogf/gf/frame/g "" DB
+//go:generate depstubber -vendor github.com/gogf/gf/database/gdb DB,Core,TX ""
+
+import (
+	"github.com/gogf/gf/database/gdb"
+	"github.com/gogf/gf/frame/g"
+)
+
+func gogfCoreTest(sql string, c *gdb.Core) {
+	c.Exec(sql, nil)               // $ querystring=sql
+	c.GetAll(sql, nil)             // $ querystring=sql
+	c.GetArray(sql, nil)           // $ querystring=sql
+	c.GetCount(sql, nil)           // $ querystring=sql
+	c.GetOne(sql, nil)             // $ querystring=sql
+	c.GetValue(sql, nil)           // $ querystring=sql
+	c.Prepare(sql, true)           // $ querystring=sql
+	c.Query(sql, nil)              // $ querystring=sql
+	c.Raw(sql, nil)                // $ querystring=sql
+	c.GetScan(nil, sql, nil)       // $ querystring=sql
+	c.GetStruct(nil, sql, nil)     // $ querystring=sql
+	c.GetStructs(nil, sql, nil)    // $ querystring=sql
+	c.DoCommit(nil, nil, sql, nil) // $ querystring=sql
+	c.DoExec(nil, nil, sql, nil)   // $ querystring=sql
+	c.DoGetAll(nil, nil, sql, nil) // $ querystring=sql
+	c.DoQuery(nil, nil, sql, nil)  // $ querystring=sql
+	c.DoPrepare(nil, nil, sql)     // $ querystring=sql
+}
+
+func gogfDbtest(sql string, c gdb.DB) {
+	c.Exec(sql, nil)               // $ querystring=sql
+	c.GetAll(sql, nil)             // $ querystring=sql
+	c.GetArray(sql, nil)           // $ querystring=sql
+	c.GetCount(sql, nil)           // $ querystring=sql
+	c.GetOne(sql, nil)             // $ querystring=sql
+	c.GetValue(sql, nil)           // $ querystring=sql
+	c.Prepare(sql, true)           // $ querystring=sql
+	c.Query(sql, nil)              // $ querystring=sql
+	c.Raw(sql, nil)                // $ querystring=sql
+	c.GetScan(nil, sql, nil)       // $ querystring=sql
+	c.DoCommit(nil, nil, sql, nil) // $ querystring=sql
+	c.DoExec(nil, nil, sql, nil)   // $ querystring=sql
+	c.DoGetAll(nil, nil, sql, nil) // $ querystring=sql
+	c.DoQuery(nil, nil, sql, nil)  // $ querystring=sql
+	c.DoPrepare(nil, nil, sql)     // $ querystring=sql
+}
+
+func gogfGTest(sql string) {
+	c := g.DB("ad")
+	c.Exec(sql, nil)               // $ querystring=sql
+	c.GetAll(sql, nil)             // $ querystring=sql
+	c.GetArray(sql, nil)           // $ querystring=sql
+	c.GetCount(sql, nil)           // $ querystring=sql
+	c.GetOne(sql, nil)             // $ querystring=sql
+	c.GetValue(sql, nil)           // $ querystring=sql
+	c.Prepare(sql, true)           // $ querystring=sql
+	c.Query(sql, nil)              // $ querystring=sql
+	c.Raw(sql, nil)                // $ querystring=sql
+	c.GetScan(nil, sql, nil)       // $ querystring=sql
+	c.DoCommit(nil, nil, sql, nil) // $ querystring=sql
+	c.DoExec(nil, nil, sql, nil)   // $ querystring=sql
+	c.DoGetAll(nil, nil, sql, nil) // $ querystring=sql
+	c.DoQuery(nil, nil, sql, nil)  // $ querystring=sql
+	c.DoPrepare(nil, nil, sql)     // $ querystring=sql
+}
+
+func main() {
+	return
+}

go/ql/test/library-tests/semmle/go/frameworks/SQL/gogf/gogf.ql

@@ -0,0 +1,4 @@
+import go
+
+from SQL::QueryString qs
+select qs

go/ql/test/library-tests/semmle/go/frameworks/SQL/gogf/vendor/github.com/gogf/gf/frame/g/stub.go

@@ -0,0 +1,14 @@
+// Code generated by depstubber. DO NOT EDIT.
+// This is a simple stub for github.com/gogf/gf/frame/g, strictly for use in testing.
+
+// See the LICENSE file for information about the licensing of the original library.
+// Source: github.com/gogf/gf/frame/g (exports: ; functions: DB)
+
+// Package g is a stub of github.com/gogf/gf/frame/g, generated by depstubber.
+package g
+
+import "github.com/gogf/gf/database/gdb"
+
+func DB(_ ...string) gdb.DB {
+	return nil
+}

go/ql/test/library-tests/semmle/go/frameworks/SQL/gogf/vendor/modules.txt

@@ -0,0 +1,57 @@
+# github.com/gogf/gf v1.16.9
+## explicit
+github.com/gogf/gf
+# github.com/BurntSushi/toml v0.3.1
+## explicit
+github.com/BurntSushi/toml
+# github.com/clbanning/mxj v1.8.5-0.20200714211355-ff02cfb8ea28
+## explicit
+github.com/clbanning/mxj
+# github.com/fatih/color v1.12.0
+## explicit
+github.com/fatih/color
+# github.com/fsnotify/fsnotify v1.4.9
+## explicit
+github.com/fsnotify/fsnotify
+# github.com/go-sql-driver/mysql v1.6.0
+## explicit
+github.com/go-sql-driver/mysql
+# github.com/gomodule/redigo v1.8.5
+## explicit
+github.com/gomodule/redigo
+# github.com/gorilla/websocket v1.4.2
+## explicit
+github.com/gorilla/websocket
+# github.com/grokify/html-strip-tags-go v0.0.1
+## explicit
+github.com/grokify/html-strip-tags-go
+# github.com/mattn/go-colorable v0.1.8
+## explicit
+github.com/mattn/go-colorable
+# github.com/mattn/go-isatty v0.0.12
+## explicit
+github.com/mattn/go-isatty
+# github.com/mattn/go-runewidth v0.0.9
+## explicit
+github.com/mattn/go-runewidth
+# github.com/olekukonko/tablewriter v0.0.5
+## explicit
+github.com/olekukonko/tablewriter
+# go.opentelemetry.io/otel v1.0.0
+## explicit
+go.opentelemetry.io/otel
+# go.opentelemetry.io/otel/trace v1.0.0
+## explicit
+go.opentelemetry.io/otel/trace
+# golang.org/x/net v0.0.0-20210520170846-37e1c6afe023
+## explicit
+golang.org/x/net
+# golang.org/x/sys v0.0.0-20210423082822-04245dca01da
+## explicit
+golang.org/x/sys
+# golang.org/x/text v0.3.6
+## explicit
+golang.org/x/text
+# gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+## explicit
+gopkg.in/yaml.v3

go/ql/test/library-tests/semmle/go/frameworks/SQL/gorqlite/go.mod

@@ -0,0 +1,5 @@
+module main
+
+go 1.18
+
+require github.com/rqlite/gorqlite v0.0.0-20220528150909-c4e99ae96be6

go/ql/test/library-tests/semmle/go/frameworks/SQL/gorqlite/gorqlite.expected

@@ -0,0 +1,6 @@
+| gorqlite.go:11:13:11:16 | sqls |
+| gorqlite.go:12:13:12:16 | sqls |
+| gorqlite.go:13:13:13:16 | sqls |
+| gorqlite.go:14:16:14:18 | sql |
+| gorqlite.go:15:16:15:18 | sql |
+| gorqlite.go:16:16:16:18 | sql |

go/ql/test/library-tests/semmle/go/frameworks/SQL/gorqlite/gorqlite.go

@@ -0,0 +1,20 @@
+package main
+
+//go:generate depstubber -vendor github.com/rqlite/gorqlite Connection Open
+
+import (
+	"github.com/rqlite/gorqlite"
+)
+
+func gorqlitetest(sql string, sqls []string) {
+	conn, _ := gorqlite.Open("dbUrl")
+	conn.Query(sqls)   // $ querystring=sqls
+	conn.Queue(sqls)   // $ querystring=sqls
+	conn.Write(sqls)   // $ querystring=sqls
+	conn.QueryOne(sql) // $ querystring=sql
+	conn.QueueOne(sql) // $ querystring=sql
+	conn.WriteOne(sql) // $ querystring=sql
+}
+func main() {
+	return
+}

go/ql/test/library-tests/semmle/go/frameworks/SQL/gorqlite/gorqlite.ql

@@ -0,0 +1,4 @@
+import go
+
+from SQL::QueryString qs
+select qs

go/ql/test/library-tests/semmle/go/frameworks/SQL/gorqlite/vendor/github.com/rqlite/gorqlite/stub.go

@@ -0,0 +1,102 @@
+// Code generated by depstubber. DO NOT EDIT.
+// This is a simple stub for github.com/rqlite/gorqlite, strictly for use in testing.
+
+// See the LICENSE file for information about the licensing of the original library.
+// Source: github.com/rqlite/gorqlite (exports: Connection; functions: Open)
+
+// Package gorqlite is a stub of github.com/rqlite/gorqlite, generated by depstubber.
+package gorqlite
+
+type Connection struct {
+	ID string
+}
+
+func (_ *Connection) Close() {}
+
+func (_ *Connection) ConsistencyLevel() (string, error) {
+	return "", nil
+}
+
+func (_ *Connection) Leader() (string, error) {
+	return "", nil
+}
+
+func (_ *Connection) Peers() ([]string, error) {
+	return nil, nil
+}
+
+func (_ *Connection) Query(_ []string) ([]QueryResult, error) {
+	return nil, nil
+}
+
+func (_ *Connection) QueryOne(_ string) (QueryResult, error) {
+	return QueryResult{}, nil
+}
+
+func (_ *Connection) Queue(_ []string) (int64, error) {
+	return 0, nil
+}
+
+func (_ *Connection) QueueOne(_ string) (int64, error) {
+	return 0, nil
+}
+
+func (_ *Connection) SetConsistencyLevel(_ string) error {
+	return nil
+}
+
+func (_ *Connection) SetExecutionWithTransaction(_ bool) error {
+	return nil
+}
+
+func (_ *Connection) Write(_ []string) ([]WriteResult, error) {
+	return nil, nil
+}
+
+func (_ *Connection) WriteOne(_ string) (WriteResult, error) {
+	return WriteResult{}, nil
+}
+
+func Open(_ string) (Connection, error) {
+	return Connection{}, nil
+}
+
+type QueryResult struct {
+	Err    error
+	Timing float64
+}
+
+func (_ *QueryResult) Columns() []string {
+	return nil
+}
+
+func (_ *QueryResult) Map() (map[string]interface{}, error) {
+	return nil, nil
+}
+
+func (_ *QueryResult) Next() bool {
+	return false
+}
+
+func (_ *QueryResult) NumRows() int64 {
+	return 0
+}
+
+func (_ *QueryResult) RowNumber() int64 {
+	return 0
+}
+
+func (_ *QueryResult) Scan(_ ...interface{}) error {
+	return nil
+}
+
+func (_ *QueryResult) Types() []string {
+	return nil
+}
+
+type WriteResult struct {
+	Err          error
+	Timing       float64
+	RowsAffected int64
+	LastInsertID int64
+}

go/ql/test/library-tests/semmle/go/frameworks/SQL/gorqlite/vendor/modules.txt

@@ -0,0 +1,3 @@
+# github.com/rqlite/gorqlite v0.0.0-20220528150909-c4e99ae96be6
+## explicit
+github.com/rqlite/gorqlite

java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt

@@ -1049,8 +1049,6 @@ open class KotlinFileExtractor(
     private val jvmOverloadsFqName = FqName("kotlin.jvm.JvmOverloads")
 
     private fun extractGeneratedOverloads(f: IrFunction, parentId: Label<out DbReftype>, maybeSourceParentId: Label<out DbReftype>?, extractBody: Boolean, extractMethodAndParameterTypeAccesses: Boolean, typeSubstitution: TypeSubstitution?, classTypeArgsIncludingOuterClasses: List<IrTypeArgument>?) {
-        if (!f.hasAnnotation(jvmOverloadsFqName))
-            return
 
         fun extractGeneratedOverload(paramList: List<IrValueParameter?>) {
             val overloadParameters = paramList.filterNotNull()
@@ -1096,6 +1094,22 @@ open class KotlinFileExtractor(
             }
         }
 
+        if (!f.hasAnnotation(jvmOverloadsFqName)) {
+            if (f is IrConstructor &&
+                f.valueParameters.isNotEmpty() &&
+                f.valueParameters.all { it.defaultValue != null } &&
+                f.parentClassOrNull?.let {
+                    // Don't create a default constructor for an annotation class, or a class that explicitly declares a no-arg constructor.
+                    !it.isAnnotationClass &&
+                    it.declarations.none { d -> d is IrConstructor && d.valueParameters.isEmpty() }
+                } == true) {
+                // Per https://kotlinlang.org/docs/classes.html#creating-instances-of-classes, a single default overload gets created specifically
+                // when we have all default parameters, regardless of `@JvmOverloads`.
+                extractGeneratedOverload(f.valueParameters.map { _ -> null })
+            }
+            return
+        }
+
         val paramList: MutableList<IrValueParameter?> = f.valueParameters.toMutableList()
         for (n in (f.valueParameters.size - 1) downTo 0) {
             if (f.valueParameters[n].defaultValue != null) {

java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt

@@ -821,7 +821,7 @@ open class KotlinUsesExtractor(
                 OperatorNameConventions.INVOKE.asString())
 
         fun getSuffixIfInternal() =
-            if (f.visibility == DescriptorVisibilities.INTERNAL && f !is IrConstructor) {
+            if (f.visibility == DescriptorVisibilities.INTERNAL && f !is IrConstructor && !(f.parent is IrFile || isExternalFileClassMember(f))) {
                 "\$" + getJvmModuleName(f)
             } else {
                 ""

java/ql/integration-tests/all-platforms/kotlin/jvmoverloads-external-class/User.java

@@ -0,0 +1,6 @@
+@AllDefaultsAnnotation
+public class User {
+
+  public static void test() { new AllDefaultsConstructor(); new AllDefaultsExplicitNoargConstructor(); }
+
+}

java/ql/integration-tests/all-platforms/kotlin/jvmoverloads-external-class/test.kt

@@ -3,3 +3,13 @@ public class Test {
   @JvmOverloads fun f(x: Int = 0, y: Int) { }
 
 }
+
+public class AllDefaultsConstructor(val x: Int = 1, val y: Int = 2) { }
+
+public annotation class AllDefaultsAnnotation(val x: Int = 1, val y: Int = 2) { }
+
+public class AllDefaultsExplicitNoargConstructor(val x: Int = 1, val y: Int = 2) {
+
+  constructor() : this(3, 4) { }
+
+}

java/ql/integration-tests/all-platforms/kotlin/jvmoverloads-external-class/test.py

@@ -1,4 +1,4 @@
 from create_database_utils import *
 
 os.mkdir('bin')
-run_codeql_database_create(["kotlinc test.kt -d bin", "kotlinc user.kt -cp bin"], lang="java")
+run_codeql_database_create(["kotlinc test.kt -d bin", "kotlinc user.kt -cp bin", "javac User.java -cp bin"], lang="java")

java/ql/lib/ext/generated/org.apache.commons.io.model.yml

@@ -1,6 +1,5 @@
- 
 # THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.
-# Definitions of taint steps in the org.apache.commons.io framework.
+# Definitions of models for the org.apache.commons.io framework.
 
 extensions:
   - addsTo:

java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll

@@ -80,54 +80,6 @@ private import internal.FlowSummaryImplSpecific as FlowSummaryImplSpecific
 private import internal.AccessPathSyntax
 private import FlowSummary
 
-/**
- * A module importing the frameworks that provide external flow data,
- * ensuring that they are visible to the taint tracking / data flow library.
- */
-private module Frameworks {
-  private import internal.ContainerFlow
-  private import semmle.code.java.frameworks.android.Android
-  private import semmle.code.java.frameworks.android.ContentProviders
-  private import semmle.code.java.frameworks.android.ExternalStorage
-  private import semmle.code.java.frameworks.android.Intent
-  private import semmle.code.java.frameworks.android.SharedPreferences
-  private import semmle.code.java.frameworks.android.Slice
-  private import semmle.code.java.frameworks.android.SQLite
-  private import semmle.code.java.frameworks.android.Widget
-  private import semmle.code.java.frameworks.ApacheHttp
-  private import semmle.code.java.frameworks.apache.Collections
-  private import semmle.code.java.frameworks.apache.Lang
-  private import semmle.code.java.frameworks.Flexjson
-  private import semmle.code.java.frameworks.generated
-  private import semmle.code.java.frameworks.guava.Guava
-  private import semmle.code.java.frameworks.jackson.JacksonSerializability
-  private import semmle.code.java.frameworks.javaee.jsf.JSFRenderer
-  private import semmle.code.java.frameworks.JaxWS
-  private import semmle.code.java.frameworks.JoddJson
-  private import semmle.code.java.frameworks.Stream
-  private import semmle.code.java.frameworks.ratpack.RatpackExec
-  private import semmle.code.java.frameworks.spring.SpringHttp
-  private import semmle.code.java.frameworks.spring.SpringWebClient
-  private import semmle.code.java.security.AndroidIntentRedirection
-  private import semmle.code.java.security.ResponseSplitting
-  private import semmle.code.java.security.InformationLeak
-  private import semmle.code.java.security.FragmentInjection
-  private import semmle.code.java.security.GroovyInjection
-  private import semmle.code.java.security.ImplicitPendingIntents
-  private import semmle.code.java.security.JndiInjection
-  private import semmle.code.java.security.LdapInjection
-  private import semmle.code.java.security.MvelInjection
-  private import semmle.code.java.security.OgnlInjection
-  private import semmle.code.java.security.TemplateInjection
-  private import semmle.code.java.security.XPath
-  private import semmle.code.java.security.XsltInjection
-  private import semmle.code.java.frameworks.Jdbc
-  private import semmle.code.java.frameworks.SpringJdbc
-  private import semmle.code.java.frameworks.MyBatis
-  private import semmle.code.java.frameworks.Hibernate
-  private import semmle.code.java.frameworks.jOOQ
-}
-
 /**
  * DEPRECATED: Define source models as data extensions instead.
  *
@@ -135,7 +87,7 @@ private module Frameworks {
  *
  * Extend this class to add additional source definitions.
  */
-class SourceModelCsv = SourceModelCsvInternal;
+deprecated class SourceModelCsv = SourceModelCsvInternal;
 
 private class SourceModelCsvInternal extends Unit {
   /** Holds if `row` specifies a source definition. */
@@ -149,7 +101,7 @@ private class SourceModelCsvInternal extends Unit {
  *
  * Extend this class to add additional sink definitions.
  */
-class SinkModelCsv = SinkModelCsvInternal;
+deprecated class SinkModelCsv = SinkModelCsvInternal;
 
 private class SinkModelCsvInternal extends Unit {
   /** Holds if `row` specifies a sink definition. */
@@ -163,7 +115,7 @@ private class SinkModelCsvInternal extends Unit {
  *
  * Extend this class to add additional flow summary definitions.
  */
-class SummaryModelCsv = SummaryModelCsvInternal;
+deprecated class SummaryModelCsv = SummaryModelCsvInternal;
 
 private class SummaryModelCsvInternal extends Unit {
   /** Holds if `row` specifies a summary definition. */
@@ -177,7 +129,7 @@ private class SummaryModelCsvInternal extends Unit {
  *
  * Extend this class to add additional negative summary definitions.
  */
-class NegativeSummaryModelCsv = NegativeSummaryModelCsvInternal;
+deprecated class NegativeSummaryModelCsv = NegativeSummaryModelCsvInternal;
 
 private class NegativeSummaryModelCsvInternal extends Unit {
   /** Holds if `row` specifies a negative summary definition. */
@@ -555,7 +507,7 @@ module ModelValidation {
       not type.regexpMatch("[a-zA-Z0-9_\\$<>]+") and
       result = "Dubious type \"" + type + "\" in " + pred + " model."
       or
-      not name.regexpMatch("[a-zA-Z0-9_]*") and
+      not name.regexpMatch("[a-zA-Z0-9_\\-]*") and
       result = "Dubious name \"" + name + "\" in " + pred + " model."
       or
       not signature.regexpMatch("|\\([a-zA-Z0-9_\\.\\$<>,\\[\\]]*\\)") and

java/ql/lib/semmle/code/java/dataflow/FlowSources.qll

@@ -36,6 +36,13 @@ abstract class RemoteFlowSource extends DataFlow::Node {
   abstract string getSourceType();
 }
 
+/**
+ * A module for importing frameworks that define remote flow sources.
+ */
+private module RemoteFlowSources {
+  private import semmle.code.java.frameworks.android.Widget
+}
+
 private class ExternalRemoteFlowSource extends RemoteFlowSource {
   ExternalRemoteFlowSource() { sourceNode(this, "remote") }
 

java/ql/lib/semmle/code/java/dataflow/FlowSteps.qll

@@ -10,17 +10,19 @@ private import semmle.code.java.dataflow.DataFlow
  * ensuring that they are visible to the taint tracking library.
  */
 private module Frameworks {
-  private import semmle.code.java.JDK
-  private import semmle.code.java.frameworks.jackson.JacksonSerializability
   private import semmle.code.java.frameworks.android.AsyncTask
   private import semmle.code.java.frameworks.android.Intent
+  private import semmle.code.java.frameworks.android.Slice
   private import semmle.code.java.frameworks.android.SQLite
-  private import semmle.code.java.frameworks.Guice
-  private import semmle.code.java.frameworks.Properties
-  private import semmle.code.java.frameworks.Protobuf
-  private import semmle.code.java.frameworks.guava.Guava
   private import semmle.code.java.frameworks.apache.Lang
   private import semmle.code.java.frameworks.ApacheHttp
+  private import semmle.code.java.frameworks.guava.Guava
+  private import semmle.code.java.frameworks.Guice
+  private import semmle.code.java.frameworks.jackson.JacksonSerializability
+  private import semmle.code.java.frameworks.Properties
+  private import semmle.code.java.frameworks.Protobuf
+  private import semmle.code.java.frameworks.ratpack.RatpackExec
+  private import semmle.code.java.JDK
 }
 
 /**

java/ql/lib/semmle/code/java/dataflow/FlowSummary.qll

@@ -6,11 +6,6 @@ import java
 private import internal.FlowSummaryImpl as Impl
 private import internal.DataFlowUtil
 
-// import all instances of SummarizedCallable below
-private module Summaries {
-  private import semmle.code.java.dataflow.ExternalFlow
-}
-
 class SummaryComponent = Impl::Public::SummaryComponent;
 
 /** Provides predicates for constructing summary components. */
@@ -102,6 +97,14 @@ abstract class SyntheticCallable extends string {
   Type getReturnType() { none() }
 }
 
+/**
+ * A module for importing frameworks that define synthetic callables.
+ */
+private module SyntheticCallables {
+  private import semmle.code.java.frameworks.android.Intent
+  private import semmle.code.java.frameworks.Stream
+}
+
 private newtype TSummarizedCallableBase =
   TSimpleCallable(Callable c) { c.isSourceDeclaration() } or
   TSyntheticCallable(SyntheticCallable c)

java/ql/lib/semmle/code/java/dataflow/TypeFlow.qll

@@ -53,16 +53,6 @@ private class TypeFlowNode extends TTypeFlowNode {
   }
 }
 
-private int getNodeKind(TypeFlowNode n) {
-  result = 1 and n instanceof TField
-  or
-  result = 2 and n instanceof TSsa
-  or
-  result = 3 and n instanceof TExpr
-  or
-  result = 4 and n instanceof TMethod
-}
-
 /** Gets `t` if it is a `RefType` or the boxed type if `t` is a primitive type. */
 private RefType boxIfNeeded(Type t) {
   t.(PrimitiveType).getBoxedType() = result or
@@ -158,107 +148,45 @@ private predicate joinStep(TypeFlowNode n1, TypeFlowNode n2) {
 
 private predicate anyStep(TypeFlowNode n1, TypeFlowNode n2) { joinStep(n1, n2) or step(n1, n2) }
 
-private import SccReduction
+private predicate sccEdge(TypeFlowNode n1, TypeFlowNode n2) { anyStep(n1, n2) and anyStep+(n2, n1) }
 
-/**
- * SCC reduction.
- *
- * This ought to be as easy as `equivalenceRelation(sccEdge/2)(n, scc)`, but
- * this HOP is not currently supported for newtypes.
- *
- * A straightforward implementation would be:
- * ```ql
- * predicate sccRepr(TypeFlowNode n, TypeFlowNode scc) {
- *  scc =
- *    max(TypeFlowNode n2 |
- *      sccEdge+(n, n2)
- *    |
- *      n2
- *      order by
- *        n2.getLocation().getStartLine(), n2.getLocation().getStartColumn(), getNodeKind(n2)
- *    )
- * }
- *
- * ```
- * but this is quadratic in the size of the SCCs.
- *
- * Instead we find local maxima by following SCC edges and determine the SCC
- * representatives from those.
- * (This is still worst-case quadratic in the size of the SCCs, but generally
- * performs better.)
- */
-private module SccReduction {
-  private predicate sccEdge(TypeFlowNode n1, TypeFlowNode n2) {
-    anyStep(n1, n2) and anyStep+(n2, n1)
-  }
+private module Scc = QlBuiltins::EquivalenceRelation<TypeFlowNode, sccEdge/2>;
 
-  private predicate sccEdgeWithMax(TypeFlowNode n1, TypeFlowNode n2, TypeFlowNode m) {
-    sccEdge(n1, n2) and
-    m =
-      max(TypeFlowNode n |
-        n = [n1, n2]
-      |
-        n order by n.getLocation().getStartLine(), n.getLocation().getStartColumn(), getNodeKind(n)
-      )
-  }
+private class TypeFlowScc = Scc::EquivalenceClass;
 
-  private predicate hasLargerNeighbor(TypeFlowNode n) {
-    exists(TypeFlowNode n2 |
-      sccEdgeWithMax(n, n2, n2) and
-      not sccEdgeWithMax(n, n2, n)
-      or
-      sccEdgeWithMax(n2, n, n2) and
-      not sccEdgeWithMax(n2, n, n)
-    )
-  }
+/** Holds if `n` is part of an SCC of size 2 or more represented by `scc`. */
+private predicate sccRepr(TypeFlowNode n, TypeFlowScc scc) { scc = Scc::getEquivalenceClass(n) }
 
-  private predicate localMax(TypeFlowNode m) {
-    sccEdgeWithMax(_, _, m) and
-    not hasLargerNeighbor(m)
-  }
-
-  private predicate sccReprFromLocalMax(TypeFlowNode scc) {
-    exists(TypeFlowNode m |
-      localMax(m) and
-      scc =
-        max(TypeFlowNode n2 |
-          sccEdge+(m, n2) and localMax(n2)
-        |
-          n2
-          order by
-            n2.getLocation().getStartLine(), n2.getLocation().getStartColumn(), getNodeKind(n2)
-        )
-    )
-  }
-
-  /** Holds if `n` is part of an SCC of size 2 or more represented by `scc`. */
-  predicate sccRepr(TypeFlowNode n, TypeFlowNode scc) {
-    sccEdge+(n, scc) and sccReprFromLocalMax(scc)
-  }
-
-  predicate sccJoinStep(TypeFlowNode n, TypeFlowNode scc) {
-    exists(TypeFlowNode mid |
-      joinStep(n, mid) and
-      sccRepr(mid, scc) and
-      not sccRepr(n, scc)
-    )
-  }
+private predicate sccJoinStep(TypeFlowNode n, TypeFlowScc scc) {
+  exists(TypeFlowNode mid |
+    joinStep(n, mid) and
+    sccRepr(mid, scc) and
+    not sccRepr(n, scc)
+  )
 }
 
-private signature predicate edgeSig(TypeFlowNode n1, TypeFlowNode n2);
+private signature class NodeSig;
 
-private signature module RankedEdge {
-  predicate edgeRank(int r, TypeFlowNode n1, TypeFlowNode n2);
+private signature module Edge {
+  class Node;
 
-  int lastRank(TypeFlowNode n);
+  predicate edge(TypeFlowNode n1, Node n2);
 }
 
-private module RankEdge<edgeSig/2 edge> implements RankedEdge {
+private signature module RankedEdge<NodeSig Node> {
+  predicate edgeRank(int r, TypeFlowNode n1, Node n2);
+
+  int lastRank(Node n);
+}
+
+private module RankEdge<Edge E> implements RankedEdge<E::Node> {
+  private import E
+
   /**
    * Holds if `r` is a ranking of the incoming edges `(n1,n2)` to `n2`. The used
    * ordering is not necessarily total, so the ranking may have gaps.
    */
-  private predicate edgeRank1(int r, TypeFlowNode n1, TypeFlowNode n2) {
+  private predicate edgeRank1(int r, TypeFlowNode n1, Node n2) {
     n1 =
       rank[r](TypeFlowNode n |
         edge(n, n2)
@@ -271,19 +199,19 @@ private module RankEdge<edgeSig/2 edge> implements RankedEdge {
    * Holds if `r2` is a ranking of the ranks from `edgeRank1`. This removes the
    * gaps from the ranking.
    */
-  private predicate edgeRank2(int r2, int r1, TypeFlowNode n) {
+  private predicate edgeRank2(int r2, int r1, Node n) {
     r1 = rank[r2](int r | edgeRank1(r, _, n) | r)
   }
 
   /** Holds if `r` is a ranking of the incoming edges `(n1,n2)` to `n2`. */
-  predicate edgeRank(int r, TypeFlowNode n1, TypeFlowNode n2) {
+  predicate edgeRank(int r, TypeFlowNode n1, Node n2) {
     exists(int r1 |
       edgeRank1(r1, n1, n2) and
       edgeRank2(r, r1, n2)
     )
   }
 
-  int lastRank(TypeFlowNode n) { result = max(int r | edgeRank(r, _, n)) }
+  int lastRank(Node n) { result = max(int r | edgeRank(r, _, n)) }
 }
 
 private signature module TypePropagation {
@@ -296,16 +224,16 @@ private signature module TypePropagation {
 }
 
 /** Implements recursion through `forall` by way of edge ranking. */
-private module ForAll<RankedEdge Edge, TypePropagation T> {
+private module ForAll<NodeSig Node, RankedEdge<Node> E, TypePropagation T> {
   /**
    * Holds if `t` is a bound that holds on one of the incoming edges to `n` and
    * thus is a candidate bound for `n`.
    */
   pragma[nomagic]
-  private predicate candJoinType(TypeFlowNode n, T::Typ t) {
+  private predicate candJoinType(Node n, T::Typ t) {
     exists(TypeFlowNode mid |
       T::candType(mid, t) and
-      Edge::edgeRank(_, mid, n)
+      E::edgeRank(_, mid, n)
     )
   }
 
@@ -314,13 +242,13 @@ private module ForAll<RankedEdge Edge, TypePropagation T> {
    * through the edges into `n` ranked from `1` to `r`.
    */
   pragma[assume_small_delta]
-  private predicate flowJoin(int r, TypeFlowNode n, T::Typ t) {
+  private predicate flowJoin(int r, Node n, T::Typ t) {
     (
       r = 1 and candJoinType(n, t)
       or
-      flowJoin(r - 1, n, t) and Edge::edgeRank(r, _, n)
+      flowJoin(r - 1, n, t) and E::edgeRank(r, _, n)
     ) and
-    forall(TypeFlowNode mid | Edge::edgeRank(r, mid, n) | T::supportsType(mid, t))
+    forall(TypeFlowNode mid | E::edgeRank(r, mid, n) | T::supportsType(mid, t))
   }
 
   /**
@@ -328,12 +256,24 @@ private module ForAll<RankedEdge Edge, TypePropagation T> {
    * coming through all the incoming edges, and therefore is a valid bound for
    * `n`.
    */
-  predicate flowJoin(TypeFlowNode n, T::Typ t) { flowJoin(Edge::lastRank(n), n, t) }
+  predicate flowJoin(Node n, T::Typ t) { flowJoin(E::lastRank(n), n, t) }
 }
 
-module RankedJoinStep = RankEdge<joinStep/2>;
+private module JoinStep implements Edge {
+  class Node = TypeFlowNode;
 
-module RankedSccJoinStep = RankEdge<sccJoinStep/2>;
+  predicate edge = joinStep/2;
+}
+
+private module SccJoinStep implements Edge {
+  class Node = TypeFlowScc;
+
+  predicate edge = sccJoinStep/2;
+}
+
+private module RankedJoinStep = RankEdge<JoinStep>;
+
+private module RankedSccJoinStep = RankEdge<SccJoinStep>;
 
 private predicate exactTypeBase(TypeFlowNode n, RefType t) {
   exists(ClassInstanceExpr e |
@@ -363,13 +303,13 @@ private predicate exactType(TypeFlowNode n, RefType t) {
   or
   // The following is an optimized version of
   // `forex(TypeFlowNode mid | joinStep(mid, n) | exactType(mid, t))`
-  ForAll<RankedJoinStep, ExactTypePropagation>::flowJoin(n, t)
+  ForAll<TypeFlowNode, RankedJoinStep, ExactTypePropagation>::flowJoin(n, t)
   or
-  exists(TypeFlowNode scc |
+  exists(TypeFlowScc scc |
     sccRepr(n, scc) and
     // Optimized version of
     // `forex(TypeFlowNode mid | sccJoinStep(mid, scc) | exactType(mid, t))`
-    ForAll<RankedSccJoinStep, ExactTypePropagation>::flowJoin(scc, t)
+    ForAll<TypeFlowScc, RankedSccJoinStep, ExactTypePropagation>::flowJoin(scc, t)
   )
 }
 
@@ -563,11 +503,11 @@ private predicate typeFlow(TypeFlowNode n, RefType t) {
   or
   exists(TypeFlowNode mid | typeFlow(mid, t) and step(mid, n))
   or
-  ForAll<RankedJoinStep, TypeFlowPropagation>::flowJoin(n, t)
+  ForAll<TypeFlowNode, RankedJoinStep, TypeFlowPropagation>::flowJoin(n, t)
   or
-  exists(TypeFlowNode scc |
+  exists(TypeFlowScc scc |
     sccRepr(n, scc) and
-    ForAll<RankedSccJoinStep, TypeFlowPropagation>::flowJoin(scc, t)
+    ForAll<TypeFlowScc, RankedSccJoinStep, TypeFlowPropagation>::flowJoin(scc, t)
   )
 }
 
@@ -703,13 +643,13 @@ private predicate hasUnionTypeFlow(TypeFlowNode n) {
   (
     // Optimized version of
     // `forex(TypeFlowNode mid | joinStep(mid, n) | unionTypeFlowBaseCand(mid, _, _) or hasUnionTypeFlow(mid))`
-    ForAll<RankedJoinStep, HasUnionTypePropagation>::flowJoin(n, _)
+    ForAll<TypeFlowNode, RankedJoinStep, HasUnionTypePropagation>::flowJoin(n, _)
     or
-    exists(TypeFlowNode scc |
+    exists(TypeFlowScc scc |
       sccRepr(n, scc) and
       // Optimized version of
       // `forex(TypeFlowNode mid | sccJoinStep(mid, scc) | unionTypeFlowBaseCand(mid, _, _) or hasUnionTypeFlow(mid))`
-      ForAll<RankedSccJoinStep, HasUnionTypePropagation>::flowJoin(scc, _)
+      ForAll<TypeFlowScc, RankedSccJoinStep, HasUnionTypePropagation>::flowJoin(scc, _)
     )
     or
     exists(TypeFlowNode mid | step(mid, n) and hasUnionTypeFlow(mid))

java/ql/lib/semmle/code/java/dataflow/internal/ContainerFlow.qll

@@ -3,7 +3,6 @@ import semmle.code.java.Collections
 import semmle.code.java.Maps
 private import semmle.code.java.dataflow.SSA
 private import DataFlowUtil
-private import semmle.code.java.dataflow.ExternalFlow
 
 private class EntryType extends RefType {
   EntryType() {

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll

@@ -244,4 +244,20 @@ module Consistency {
     not callable = viableCallable(call) and
     not any(ConsistencyConfiguration c).viableImplInCallContextTooLargeExclude(call, ctx, callable)
   }
+
+  query predicate uniqueParameterNodeAtPosition(
+    DataFlowCallable c, ParameterPosition pos, Node p, string msg
+  ) {
+    isParameterNode(p, c, pos) and
+    not exists(unique(Node p0 | isParameterNode(p0, c, pos))) and
+    msg = "Parameters with overlapping positions."
+  }
+
+  query predicate uniqueParameterNodePosition(
+    DataFlowCallable c, ParameterPosition pos, Node p, string msg
+  ) {
+    isParameterNode(p, c, pos) and
+    not exists(unique(ParameterPosition pos0 | isParameterNode(p, c, pos0))) and
+    msg = "Parameter node with multiple positions."
+  }
 }

java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll

@@ -520,7 +520,8 @@ module Private {
   predicate summaryParameterNodeRange(SummarizedCallable c, ParameterPosition pos) {
     parameterReadState(c, _, pos)
     or
-    isParameterPostUpdate(_, c, pos)
+    // Same as `isParameterPostUpdate(_, c, pos)`, but can be used in a negative context
+    any(SummaryNodeState state).isOutputState(c, SummaryComponentStack::argument(pos))
   }
 
   private predicate callbackOutput(

java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImplSpecific.qll

@@ -14,6 +14,13 @@ private import semmle.code.java.dataflow.internal.AccessPathSyntax as AccessPath
 
 class SummarizedCallableBase = FlowSummary::SummarizedCallableBase;
 
+/**
+ * A module for importing frameworks that define synthetic globals.
+ */
+private module SyntheticGlobals {
+  private import semmle.code.java.frameworks.android.Intent
+}
+
 DataFlowCallable inject(SummarizedCallable c) { result.asSummarizedCallable() = c }
 
 /** Gets the parameter position of the instance parameter. */

java/ql/lib/semmle/code/java/dataflow/internal/TaintTrackingUtil.qll

@@ -10,7 +10,6 @@ private import semmle.code.java.dataflow.internal.ContainerFlow
 private import semmle.code.java.frameworks.spring.SpringController
 private import semmle.code.java.frameworks.spring.SpringHttp
 private import semmle.code.java.frameworks.Networking
-private import semmle.code.java.dataflow.ExternalFlow
 private import semmle.code.java.dataflow.FlowSources
 private import semmle.code.java.dataflow.internal.DataFlowPrivate
 import semmle.code.java.dataflow.FlowSteps

java/ql/lib/semmle/code/java/frameworks/ApacheHttp.qll

@@ -4,7 +4,6 @@
 
 import java
 private import semmle.code.java.dataflow.FlowSteps
-private import semmle.code.java.dataflow.ExternalFlow
 
 class ApacheHttpGetParams extends Method {
   ApacheHttpGetParams() {

java/ql/lib/semmle/code/java/frameworks/Flexjson.qll

@@ -3,7 +3,6 @@
  */
 
 import java
-private import semmle.code.java.dataflow.ExternalFlow
 
 /** The class `flexjson.JSONDeserializer`. */
 class FlexjsonDeserializer extends RefType {

java/ql/lib/semmle/code/java/frameworks/Hibernate.qll

@@ -3,7 +3,6 @@
  */
 
 import java
-private import semmle.code.java.dataflow.ExternalFlow
 
 /** The interface `org.hibernate.query.QueryProducer`. */
 class HibernateQueryProducer extends RefType {

java/ql/lib/semmle/code/java/frameworks/JaxWS.qll

@@ -4,7 +4,6 @@
  */
 
 import java
-private import semmle.code.java.dataflow.ExternalFlow
 private import semmle.code.java.security.XSS
 
 /**

java/ql/lib/semmle/code/java/frameworks/Jdbc.qll

@@ -2,7 +2,7 @@
  * Provides classes and predicates for working with the Java JDBC API.
  */
 
-private import semmle.code.java.dataflow.ExternalFlow
+import java
 
 /*--- Types ---*/
 /** The interface `java.sql.Connection`. */

java/ql/lib/semmle/code/java/frameworks/JoddJson.qll

@@ -3,7 +3,6 @@
  */
 
 import java
-private import semmle.code.java.dataflow.ExternalFlow
 
 /** The class `jodd.json.Parser`. */
 class JoddJsonParser extends RefType {

java/ql/lib/semmle/code/java/frameworks/MyBatis.qll

@@ -5,7 +5,6 @@
 import java
 private import semmle.code.java.dataflow.DataFlow
 private import semmle.code.java.dataflow.TaintTracking
-private import semmle.code.java.dataflow.ExternalFlow
 
 /** The class `org.apache.ibatis.jdbc.SqlRunner`. */
 class MyBatisSqlRunner extends RefType {

java/ql/lib/semmle/code/java/frameworks/Regex.qll

@@ -1,6 +1,6 @@
 /** Definitions related to `java.util.regex`. */
 
-private import semmle.code.java.dataflow.ExternalFlow
+import java
 
 /** The class `java.util.regex.Pattern`. */
 class TypeRegexPattern extends Class {

java/ql/lib/semmle/code/java/frameworks/SpringJdbc.qll

@@ -3,7 +3,6 @@
  */
 
 import java
-private import semmle.code.java.dataflow.ExternalFlow
 
 /** The class `org.springframework.jdbc.core.JdbcTemplate`. */
 class JdbcTemplate extends RefType {

java/ql/lib/semmle/code/java/frameworks/Stream.qll

@@ -1,6 +1,5 @@
 /** Definitions related to `java.util.stream`. */
 
-private import semmle.code.java.dataflow.ExternalFlow
 private import semmle.code.java.dataflow.FlowSummary
 
 private class CollectCall extends MethodAccess {

java/ql/lib/semmle/code/java/frameworks/android/Android.qll

@@ -3,7 +3,6 @@
  */
 
 import java
-private import semmle.code.java.dataflow.ExternalFlow
 private import semmle.code.xml.AndroidManifest
 
 /**

java/ql/lib/semmle/code/java/frameworks/android/ContentProviders.qll

@@ -3,7 +3,6 @@
  */
 
 import java
-private import semmle.code.java.dataflow.ExternalFlow
 
 /** The class `android.content.ContentValues`. */
 class ContentValues extends Class {

java/ql/lib/semmle/code/java/frameworks/android/SQLite.qll

@@ -1,7 +1,6 @@
 /** Provides classes and predicates for working with SQLite databases. */
 
 import java
-private import semmle.code.java.dataflow.ExternalFlow
 private import semmle.code.java.dataflow.FlowSteps
 private import semmle.code.java.frameworks.android.Android
 

java/ql/lib/semmle/code/java/frameworks/android/SharedPreferences.qll

@@ -1,7 +1,6 @@
 /** Provides classes related to `android.content.SharedPreferences`. */
 
 import java
-private import semmle.code.java.dataflow.ExternalFlow
 
 /** The interface `android.content.SharedPreferences`. */
 class SharedPreferences extends Interface {

java/ql/lib/semmle/code/java/frameworks/android/Slice.qll

@@ -3,7 +3,6 @@
 import java
 private import semmle.code.java.dataflow.DataFlow
 private import semmle.code.java.dataflow.FlowSteps
-private import semmle.code.java.dataflow.ExternalFlow
 
 /** The class `androidx.slice.SliceProvider`. */
 class SliceProvider extends Class {

java/ql/lib/semmle/code/java/frameworks/apache/Collections.qll

@@ -2,7 +2,6 @@
 
 import java
 private import semmle.code.java.dataflow.FlowSteps
-private import semmle.code.java.dataflow.ExternalFlow
 
 /**
  * The method `isEmpty` in either `org.apache.commons.collections.CollectionUtils`

java/ql/lib/semmle/code/java/frameworks/apache/Lang.qll

@@ -2,7 +2,6 @@
 
 import java
 private import semmle.code.java.dataflow.FlowSteps
-private import semmle.code.java.dataflow.ExternalFlow
 
 /**
  * The class `org.apache.commons.lang.RandomStringUtils` or `org.apache.commons.lang3.RandomStringUtils`.

java/ql/lib/semmle/code/java/frameworks/generated.qll

@@ -1,9 +0,0 @@
-/**
- * A module importing all generated Models as Data models.
- */
-
-import java
-
-private module GeneratedFrameworks {
-  private import kotlin.StdLibGenerated
-}

java/ql/lib/semmle/code/java/frameworks/guava/Collections.qll

@@ -3,7 +3,6 @@
 import java
 private import semmle.code.java.dataflow.DataFlow
 private import semmle.code.java.dataflow.FlowSteps
-private import semmle.code.java.dataflow.ExternalFlow
 private import semmle.code.java.Collections
 
 private string guavaCollectPackage() { result = "com.google.common.collect" }

java/ql/lib/semmle/code/java/frameworks/jOOQ.qll

@@ -3,7 +3,6 @@
  */
 
 import java
-private import semmle.code.java.dataflow.ExternalFlow
 
 /**
  * Methods annotated with this allow for generation of "plain SQL"

java/ql/lib/semmle/code/java/frameworks/jackson/JacksonSerializability.qll

@@ -9,7 +9,6 @@ import semmle.code.java.Reflection
 import semmle.code.java.dataflow.DataFlow
 private import semmle.code.java.dataflow.internal.DataFlowForSerializability
 import semmle.code.java.dataflow.FlowSteps
-private import semmle.code.java.dataflow.ExternalFlow
 
 /**
  * A `@com.fasterxml.jackson.annotation.JsonIgnore` annoation.

java/ql/lib/semmle/code/java/frameworks/javaee/jsf/JSFRenderer.qll

@@ -1,7 +1,6 @@
 /** Provides classes and predicates for working with JavaServer Faces renderer. */
 
 import java
-private import semmle.code.java.dataflow.ExternalFlow
 
 /**
  * The JSF class `FacesContext` for processing HTTP requests.

java/ql/lib/semmle/code/java/frameworks/ratpack/RatpackExec.qll

@@ -5,7 +5,6 @@
 import java
 private import semmle.code.java.dataflow.DataFlow
 private import semmle.code.java.dataflow.FlowSteps
-private import semmle.code.java.dataflow.ExternalFlow
 
 /** A reference type that extends a parameterization the Promise type. */
 private class RatpackPromise extends RefType {

java/ql/lib/semmle/code/java/frameworks/spring/SpringHttp.qll

@@ -4,7 +4,6 @@
  */
 
 import java
-private import semmle.code.java.dataflow.ExternalFlow
 private import semmle.code.java.dataflow.DataFlow
 private import semmle.code.java.frameworks.spring.SpringController
 private import semmle.code.java.security.XSS as XSS