Merge branch 'main' into kaeluka/java-automodel-variadic-args

cpp/downgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/upgrade.properties

@@ -0,0 +1,2 @@
+description: Remove _Float128 type
+compatibility: full

cpp/ql/lib/change-notes/2023-08-07-removal-of-float128x.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The `_Float128x` type is no longer exposed as a builtin type. As this type could not occur any code base, this should only affect queries that explicitly looked at the builtin types.

cpp/ql/lib/semmle/code/cpp/Type.qll

@@ -814,9 +814,6 @@ private predicate floatingPointTypeMapping(
   // _Float128
   kind = 49 and base = 2 and domain = TRealDomain() and realKind = 49 and extended = false
   or
-  // _Float128x
-  kind = 50 and base = 2 and domain = TRealDomain() and realKind = 50 and extended = true
-  or
   // _Float16
   kind = 52 and base = 2 and domain = TRealDomain() and realKind = 52 and extended = false
   or

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll

@@ -1,3 +1,3 @@
 private import DataFlowImplSpecific
-private import codeql.dataflow.DataFlowImpl
+private import codeql.dataflow.internal.DataFlowImpl
 import MakeImpl<CppOldDataFlow>

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll

@@ -1,3 +1,3 @@
 private import DataFlowImplSpecific
-private import codeql.dataflow.DataFlowImplCommon
+private import codeql.dataflow.internal.DataFlowImplCommon
 import MakeImplCommon<CppOldDataFlow>

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplSpecific.qll

@@ -2,7 +2,7 @@
  * Provides C++-specific definitions for use in the data flow library.
  */
 
-private import codeql.dataflow.DataFlowParameter
+private import codeql.dataflow.DataFlow
 
 module Private {
   import DataFlowPrivate
@@ -13,7 +13,7 @@ module Public {
   import DataFlowUtil
 }
 
-module CppOldDataFlow implements DataFlowParameter {
+module CppOldDataFlow implements InputSig {
   import Private
   import Public
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll

@@ -1,3 +1,3 @@
 private import DataFlowImplSpecific
-private import codeql.dataflow.DataFlowImpl
+private import codeql.dataflow.internal.DataFlowImpl
 import MakeImpl<CppDataFlow>

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll

@@ -1,3 +1,3 @@
 private import DataFlowImplSpecific
-private import codeql.dataflow.DataFlowImplCommon
+private import codeql.dataflow.internal.DataFlowImplCommon
 import MakeImplCommon<CppDataFlow>

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplSpecific.qll

@@ -2,7 +2,7 @@
  * Provides IR-specific definitions for use in the data flow library.
  */
 
-private import codeql.dataflow.DataFlowParameter
+private import codeql.dataflow.DataFlow
 
 module Private {
   import DataFlowPrivate
@@ -13,7 +13,7 @@ module Public {
   import DataFlowUtil
 }
 
-module CppDataFlow implements DataFlowParameter {
+module CppDataFlow implements InputSig {
   import Private
   import Public
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll

@@ -1078,7 +1078,7 @@ private IRVariable getIRVariableForParameterNode(ParameterNode p) {
 
 /** Holds if `v` is the source variable corresponding to the parameter represented by `p`. */
 pragma[nomagic]
-private predicate parameterNodeHasSourceVariable(ParameterNode p, Ssa::SourceIRVariable v) {
+private predicate parameterNodeHasSourceVariable(ParameterNode p, Ssa::SourceVariable v) {
   v.getIRVariable() = getIRVariableForParameterNode(p) and
   exists(Position pos | p.isParameterOf(_, pos) |
     pos instanceof DirectPosition and

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll

@@ -781,26 +781,12 @@ class IndirectArgumentOutNode extends Node, TIndirectArgumentOutNode, PartialDef
   override Expr getDefinedExpr() { result = operand.getDef().getUnconvertedResultExpression() }
 }
 
-pragma[nomagic]
-predicate indirectReturnOutNodeOperand0(CallInstruction call, Operand operand, int indirectionIndex) {
-  Ssa::hasRawIndirectInstruction(call, indirectionIndex) and
-  operandForFullyConvertedCall(operand, call)
-}
-
-pragma[nomagic]
-predicate indirectReturnOutNodeInstruction0(
-  CallInstruction call, Instruction instr, int indirectionIndex
-) {
-  Ssa::hasRawIndirectInstruction(call, indirectionIndex) and
-  instructionForFullyConvertedCall(instr, call)
-}
-
 /**
  * Holds if `node` is an indirect operand with columns `(operand, indirectionIndex)`, and
  * `operand` represents a use of the fully converted value of `call`.
  */
 private predicate hasOperand(Node node, CallInstruction call, int indirectionIndex, Operand operand) {
-  indirectReturnOutNodeOperand0(call, operand, indirectionIndex) and
+  operandForFullyConvertedCall(operand, call) and
   hasOperandAndIndex(node, operand, indirectionIndex)
 }
 
@@ -813,7 +799,7 @@ private predicate hasOperand(Node node, CallInstruction call, int indirectionInd
 private predicate hasInstruction(
   Node node, CallInstruction call, int indirectionIndex, Instruction instr
 ) {
-  indirectReturnOutNodeInstruction0(call, instr, indirectionIndex) and
+  instructionForFullyConvertedCall(instr, call) and
   hasInstructionAndIndex(node, instr, indirectionIndex)
 }
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternals.qll

@@ -10,32 +10,35 @@ private import ssa0.SsaInternals as SsaInternals0
 import SsaInternalsCommon
 
 private module SourceVariables {
-  int getMaxIndirectionForIRVariable(IRVariable var) {
-    exists(Type type, boolean isGLValue |
-      var.getLanguageType().hasType(type, isGLValue) and
-      if isGLValue = true
-      then result = 1 + getMaxIndirectionsForType(type)
-      else result = getMaxIndirectionsForType(type)
-    )
-  }
-
   cached
   private newtype TSourceVariable =
-    TSourceIRVariable(BaseIRVariable baseVar, int ind) {
-      ind = [0 .. getMaxIndirectionForIRVariable(baseVar.getIRVariable())]
-    } or
-    TCallVariable(AllocationInstruction call, int ind) {
-      ind = [0 .. countIndirectionsForCppType(getResultLanguageType(call))]
+    TMkSourceVariable(SsaInternals0::SourceVariable base, int ind) {
+      ind = [0 .. countIndirectionsForCppType(base.getLanguageType()) + 1]
     }
 
-  abstract class SourceVariable extends TSourceVariable {
+  class SourceVariable extends TSourceVariable {
+    SsaInternals0::SourceVariable base;
     int ind;
 
-    bindingset[ind]
-    SourceVariable() { any() }
+    SourceVariable() { this = TMkSourceVariable(base, ind) }
+
+    /** Gets the IR variable associated with this `SourceVariable`, if any. */
+    IRVariable getIRVariable() { result = base.(BaseIRVariable).getIRVariable() }
+
+    /**
+     * Gets the base source variable (i.e., the variable without any
+     * indirections) of this source variable.
+     */
+    SsaInternals0::SourceVariable getBaseVariable() { result = base }
 
     /** Gets a textual representation of this element. */
-    abstract string toString();
+    string toString() {
+      ind = 0 and
+      result = this.getBaseVariable().toString()
+      or
+      ind > 0 and
+      result = this.getBaseVariable().toString() + " indirection"
+    }
 
     /**
      * Gets the number of loads performed on the base source variable
@@ -43,65 +46,19 @@ private module SourceVariables {
      */
     int getIndirection() { result = ind }
 
-    /**
-     * Gets the base source variable (i.e., the variable without any
-     * indirections) of this source variable.
-     */
-    abstract BaseSourceVariable getBaseVariable();
-
     /** Holds if this variable is a glvalue. */
-    predicate isGLValue() { none() }
+    predicate isGLValue() { ind = 0 }
 
     /**
      * Gets the type of this source variable. If `isGLValue()` holds, then
      * the type of this source variable should be thought of as "pointer
      * to `getType()`".
      */
-    abstract DataFlowType getType();
-  }
-
-  class SourceIRVariable extends SourceVariable, TSourceIRVariable {
-    BaseIRVariable var;
-
-    SourceIRVariable() { this = TSourceIRVariable(var, ind) }
-
-    IRVariable getIRVariable() { result = var.getIRVariable() }
-
-    override BaseIRVariable getBaseVariable() { result.getIRVariable() = this.getIRVariable() }
-
-    override string toString() {
-      ind = 0 and
-      result = this.getIRVariable().toString()
-      or
-      ind > 0 and
-      result = this.getIRVariable().toString() + " indirection"
+    DataFlowType getType() {
+      if this.isGLValue()
+      then result = base.getType()
+      else result = getTypeImpl(base.getType(), ind - 1)
     }
-
-    override predicate isGLValue() { ind = 0 }
-
-    override DataFlowType getType() {
-      if ind = 0 then result = var.getType() else result = getTypeImpl(var.getType(), ind - 1)
-    }
-  }
-
-  class CallVariable extends SourceVariable, TCallVariable {
-    AllocationInstruction call;
-
-    CallVariable() { this = TCallVariable(call, ind) }
-
-    AllocationInstruction getCall() { result = call }
-
-    override BaseCallVariable getBaseVariable() { result.getCallInstruction() = call }
-
-    override string toString() {
-      ind = 0 and
-      result = "Call"
-      or
-      ind > 0 and
-      result = "Call indirection"
-    }
-
-    override DataFlowType getType() { result = getTypeImpl(call.getResultType(), ind) }
   }
 }
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll

@@ -370,15 +370,20 @@ newtype TBaseSourceVariable =
   // Each allocation gets its own source variable
   TBaseCallVariable(AllocationInstruction call)
 
-abstract class BaseSourceVariable extends TBaseSourceVariable {
+abstract private class AbstractBaseSourceVariable extends TBaseSourceVariable {
   /** Gets a textual representation of this element. */
   abstract string toString();
 
   /** Gets the type of this base source variable. */
-  abstract DataFlowType getType();
+  final DataFlowType getType() { this.getLanguageType().hasUnspecifiedType(result, _) }
+
+  /** Gets the `CppType` of this base source variable. */
+  abstract CppType getLanguageType();
 }
 
-class BaseIRVariable extends BaseSourceVariable, TBaseIRVariable {
+final class BaseSourceVariable = AbstractBaseSourceVariable;
+
+class BaseIRVariable extends AbstractBaseSourceVariable, TBaseIRVariable {
   IRVariable var;
 
   IRVariable getIRVariable() { result = var }
@@ -387,10 +392,10 @@ class BaseIRVariable extends BaseSourceVariable, TBaseIRVariable {
 
   override string toString() { result = var.toString() }
 
-  override DataFlowType getType() { result = var.getType() }
+  override CppType getLanguageType() { result = var.getLanguageType() }
 }
 
-class BaseCallVariable extends BaseSourceVariable, TBaseCallVariable {
+class BaseCallVariable extends AbstractBaseSourceVariable, TBaseCallVariable {
   AllocationInstruction call;
 
   BaseCallVariable() { this = TBaseCallVariable(call) }
@@ -399,7 +404,7 @@ class BaseCallVariable extends BaseSourceVariable, TBaseCallVariable {
 
   override string toString() { result = call.toString() }
 
-  override DataFlowType getType() { result = call.getResultType() }
+  override CppType getLanguageType() { result = getResultLanguageType(call) }
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/ssa0/SsaInternals.qll

@@ -15,15 +15,12 @@ private import semmle.code.cpp.ir.dataflow.internal.DataFlowUtil
 private import semmle.code.cpp.ir.dataflow.internal.SsaInternalsCommon
 
 private module SourceVariables {
-  class SourceVariable instanceof BaseSourceVariable {
-    string toString() { result = BaseSourceVariable.super.toString() }
-
+  class SourceVariable extends BaseSourceVariable {
+    /**
+     * Gets the base source variable of this `SourceVariable`.
+     */
     BaseSourceVariable getBaseVariable() { result = this }
   }
-
-  class SourceIRVariable = BaseIRVariable;
-
-  class CallVariable = BaseCallVariable;
 }
 
 import SourceVariables

cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticExpr.qll

@@ -307,3 +307,8 @@ class SemConditionalExpr extends SemKnownExpr {
     branch = false and result = falseResult
   }
 }
+
+/** Holds if `upper = true` and `e <= bound` or `upper = false` and `e >= bound`. */
+predicate semHasConstantBoundConstantSpecific(SemExpr e, float bound, boolean upper) {
+  Specific::hasConstantBoundConstantSpecific(e, bound, upper)
+}

cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticExprSpecific.qll

@@ -434,6 +434,50 @@ module SemanticExprConfig {
 
   /** Gets the expression associated with `instr`. */
   SemExpr getSemanticExpr(IR::Instruction instr) { result = Equiv::getEquivalenceClass(instr) }
+
+  private predicate typeBounds(SemType t, float lb, float ub) {
+    exists(SemIntegerType integralType, float limit |
+      integralType = t and limit = 2.pow(8 * integralType.getByteSize())
+    |
+      if integralType instanceof SemBooleanType
+      then lb = 0 and ub = 1
+      else
+        if integralType.isSigned()
+        then (
+          lb = -(limit / 2) and ub = (limit / 2) - 1
+        ) else (
+          lb = 0 and ub = limit - 1
+        )
+    )
+    or
+    // This covers all floating point types. The range is (-Inf, +Inf).
+    t instanceof SemFloatingPointType and lb = -(1.0 / 0.0) and ub = 1.0 / 0.0
+  }
+
+  /**
+   * Holds if `upper = true` and `e <= bound` or `upper = false` and `e >= bound` based
+   * only on type information.
+   */
+  predicate hasConstantBoundConstantSpecific(Expr e, float bound, boolean upper) {
+    exists(
+      SemType converted, SemType unconverted, float unconvertedLb, float convertedLb,
+      float unconvertedUb, float convertedUb
+    |
+      unconverted = getSemanticType(e.getUnconverted().getResultIRType()) and
+      converted = getSemanticType(e.getConverted().getResultIRType()) and
+      typeBounds(unconverted, unconvertedLb, unconvertedUb) and
+      typeBounds(converted, convertedLb, convertedUb) and
+      (
+        upper = true and
+        unconvertedUb < convertedUb and
+        bound = unconvertedUb
+        or
+        upper = false and
+        unconvertedLb > convertedLb and
+        bound = unconvertedLb
+      )
+    )
+  }
 }
 
 predicate getSemanticExpr = SemanticExprConfig::getSemanticExpr/1;
@@ -457,3 +501,5 @@ IRBound::Bound getCppBound(SemBound bound) { bound = result }
 SemGuard getSemanticGuard(IRGuards::IRGuardCondition guard) { result = guard }
 
 IRGuards::IRGuardCondition getCppGuard(SemGuard guard) { guard = result }
+
+predicate hasConstantBoundConstantSpecific = SemanticExprConfig::hasConstantBoundConstantSpecific/3;

cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisConstantSpecific.qll

@@ -74,7 +74,10 @@ module CppLangImplConstant implements LangSig<FloatDelta> {
   /**
    * Holds if `e >= bound` (if `upper = false`) or `e <= bound` (if `upper = true`).
    */
-  predicate hasConstantBound(SemExpr e, float bound, boolean upper) { none() }
+  predicate hasConstantBound(SemExpr e, float bound, boolean upper, SemReason reason) {
+    semHasConstantBoundConstantSpecific(e, bound, upper) and
+    reason instanceof SemTypeReason
+  }
 
   /**
    * Holds if `e >= bound + delta` (if `upper = false`) or `e <= bound + delta` (if `upper = true`).

cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisRelativeSpecific.qll

@@ -110,7 +110,7 @@ module CppLangImplRelative implements LangSig<FloatDelta> {
   /**
    * Holds if `e >= bound` (if `upper = false`) or `e <= bound` (if `upper = true`).
    */
-  predicate hasConstantBound(SemExpr e, float bound, boolean upper) { none() }
+  predicate hasConstantBound(SemExpr e, float bound, boolean upper, SemReason reason) { none() }
 
   /**
    * Holds if `e >= bound + delta` (if `upper = false`) or `e <= bound + delta` (if `upper = true`).

cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisStage.qll

@@ -155,7 +155,7 @@ signature module LangSig<DeltaSig D> {
   /**
    * Holds if `e >= bound` (if `upper = false`) or `e <= bound` (if `upper = true`).
    */
-  predicate hasConstantBound(SemExpr e, D::Delta bound, boolean upper);
+  predicate hasConstantBound(SemExpr e, D::Delta bound, boolean upper, SemReason reason);
 
   /**
    * Holds if `e >= bound + delta` (if `upper = false`) or `e <= bound + delta` (if `upper = true`).
@@ -920,14 +920,15 @@ module RangeStage<
    * Holds if `e` has an upper (for `upper = true`) or lower
    * (for `upper = false`) bound of `b`.
    */
-  private predicate baseBound(SemExpr e, D::Delta b, boolean upper) {
-    hasConstantBound(e, b, upper)
+  private predicate baseBound(SemExpr e, D::Delta b, boolean upper, SemReason reason) {
+    hasConstantBound(e, b, upper, reason)
     or
     upper = false and
     b = D::fromInt(0) and
     semPositive(e.(SemBitAndExpr).getAnOperand()) and
     // REVIEW: We let the language opt out here to preserve original results.
-    not ignoreZeroLowerBound(e)
+    not ignoreZeroLowerBound(e) and
+    reason instanceof SemNoReason
   }
 
   /**
@@ -1055,11 +1056,10 @@ module RangeStage<
       origdelta = delta and
       reason instanceof SemNoReason
       or
-      baseBound(e, delta, upper) and
+      baseBound(e, delta, upper, reason) and
       b instanceof SemZeroBound and
       fromBackEdge = false and
-      origdelta = delta and
-      reason instanceof SemNoReason
+      origdelta = delta
       or
       exists(SemSsaVariable v, SemSsaReadPositionBlock bb |
         boundedSsa(v, bb, b, delta, upper, fromBackEdge, origdelta, reason) and

cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/RangeAnalysisUtil.qll

@@ -20,7 +20,8 @@ private Instruction getABoundIn(SemBound b, IRFunction func) {
 pragma[inline]
 private predicate boundedImpl(Instruction i, Instruction b, int delta) {
   exists(SemBound bound, IRFunction func |
-    semBounded(getSemanticExpr(i), bound, delta, true, _) and
+    semBounded(getSemanticExpr(i), bound, delta, true,
+      any(SemReason reason | not reason instanceof SemTypeReason)) and
     b = getABoundIn(bound, func) and
     i.getEnclosingIRFunction() = func
   )

cpp/ql/lib/semmlecode.cpp.dbscheme

@@ -608,7 +608,7 @@ case @builtintype.kind of
 | 47 = @std_float64           // _Float64
 | 48 = @float64x              // _Float64x
 | 49 = @std_float128          // _Float128
-| 50 = @float128x             // _Float128x
+// ... 50 _Float128x
 | 51 = @char8_t
 | 52 = @float16               // _Float16
 | 53 = @complex_float16       // _Complex _Float16

cpp/ql/lib/upgrades/19887dbd33327fb07d54251786e0cb2578539775/builtintypes.ql

@@ -0,0 +1,13 @@
+class BuiltinType extends @builtintype {
+  string toString() { none() }
+}
+
+predicate isFloat128xBuiltinType(BuiltinType type) {
+  exists(int kind | builtintypes(type, _, kind, _, _, _) | kind = 50)
+}
+
+from BuiltinType type, string name, int kind, int kind_new, int size, int sign, int alignment
+where
+  builtintypes(type, name, kind, size, sign, alignment) and
+  if isFloat128xBuiltinType(type) then kind_new = 1 else kind_new = kind
+select type, name, kind_new, size, sign, alignment

cpp/ql/lib/upgrades/19887dbd33327fb07d54251786e0cb2578539775/upgrade.properties

@@ -0,0 +1,3 @@
+description: Remove _Float128 type
+compatibility: partial
+builtintypes.rel: run builtintypes.qlo

cpp/ql/src/experimental/Security/CWE/CWE-193/ConstantSizeArrayOffByOne.ql

@@ -28,7 +28,8 @@ Instruction getABoundIn(SemBound b, IRFunction func) {
 pragma[inline]
 predicate boundedImpl(Instruction i, Instruction b, int delta) {
   exists(SemBound bound, IRFunction func |
-    semBounded(getSemanticExpr(i), bound, delta, true, _) and
+    semBounded(getSemanticExpr(i), bound, delta, true,
+      any(SemReason reason | not reason instanceof SemTypeReason)) and
     b = getABoundIn(bound, func) and
     pragma[only_bind_out](i.getEnclosingIRFunction()) = func
   )
@@ -93,7 +94,8 @@ predicate arrayTypeHasSizes(ArrayType arr, int baseTypeSize, int size) {
 bindingset[pai]
 pragma[inline_late]
 predicate constantUpperBounded(PointerArithmeticInstruction pai, int delta) {
-  semBounded(getSemanticExpr(pai.getRight()), any(SemZeroBound b), delta, true, _)
+  semBounded(getSemanticExpr(pai.getRight()), any(SemZeroBound b), delta, true,
+    any(SemReason reason | not reason instanceof SemTypeReason))
 }
 
 bindingset[pai, size]

cpp/ql/test/library-tests/ir/range-analysis/SimpleRangeAnalysis_tests.cpp

@@ -195,18 +195,18 @@ int test13(char c, int i) {
   int z = i+1;  // $ overflow=+
   range(z); // $ range="==InitializeParameter: i+1"
   range(c + i + uc + x + y + z); // $ overflow=+- overflow=+ overflow=- MISSING: range=>=1
-  range((double)(c + i + uc + x + y + z)); // $ overflow=+ overflow=+- overflow=- MISSING:  range=>=1
+  range((double)(c + i + uc + x + y + z)); // $ overflow=+ overflow=+- overflow=- range=<=4294967295 MISSING: range=>=1
   return (double)(c + i + uc + x + y + z);  // $ overflow=+- overflow=+ overflow=-
 }
 
 // Regression test for ODASA-6013.
 int test14(int x) {
   int x0 = (int)(char)x;
-  range(x0);
+  range(x0); // $ range=<=127 range=>=-128
   int x1 = (int)(unsigned char)x;
-  range(x1);
+  range(x1); // $ range=<=255 range=>=0
   int x2 = (int)(unsigned short)x;
-  range(x2);
+  range(x2); // $ range=<=65535 range=>=0
   int x3 = (int)(unsigned int)x;
   range(x3);
   char c0 = x;
@@ -759,9 +759,9 @@ unsigned long mult_overflow() {
 unsigned long mult_lower_bound(unsigned int ui, unsigned long ul) {
   if (ui >= 10) {
     range(ui); // $ range=>=10
-    range((unsigned long)ui); // $ range=>=10
-    unsigned long result = (unsigned long)ui * ui; // $ overflow=+
-    range(result); // $ MISSING: range=>=100
+    range((unsigned long)ui); // $ range=>=10 range=<=4294967295
+    unsigned long result = (unsigned long)ui * ui; // no overflow
+    range(result); // $ range=>=100 range=<=18446744065119617024
     return result; // BUG: upper bound should be >= 18446744065119617025
   }
   if (ul >= 10) {
@@ -888,7 +888,7 @@ void notequal_variations(short n, float f) {
   }
 
   if (n >= 5) {
-    if (2 * n - 10 == 0) { // $ overflow=+
+    if (2 * n - 10 == 0) { // no overflow
       range(n); // $ range=>=5 MISSING: range===5
       return;
     }
@@ -936,7 +936,7 @@ void two_bounds_from_one_test(short ss, unsigned short us) {
     range(ss); // -32768 .. 32767
   }
 
-  if (ss + 1 < sizeof(int)) {  // $ overflow=+
+  if (ss + 1 < sizeof(int)) { // $ overflow=-
     range(ss); // -1 .. 2
   }
 }

cpp/ql/test/library-tests/templates/type_instantiations/types.expected

@@ -13,7 +13,6 @@
 | file://:0:0:0:0 | _Float64 |
 | file://:0:0:0:0 | _Float64x |
 | file://:0:0:0:0 | _Float128 |
-| file://:0:0:0:0 | _Float128x |
 | file://:0:0:0:0 | _Imaginary double |
 | file://:0:0:0:0 | _Imaginary float |
 | file://:0:0:0:0 | _Imaginary long double |

cpp/ql/test/library-tests/type_sizes/type_sizes.expected

@@ -33,7 +33,6 @@
 | file://:0:0:0:0 | _Float64 | 8 |
 | file://:0:0:0:0 | _Float64x | 16 |
 | file://:0:0:0:0 | _Float128 | 16 |
-| file://:0:0:0:0 | _Float128x | 32 |
 | file://:0:0:0:0 | _Imaginary double | 8 |
 | file://:0:0:0:0 | _Imaginary float | 4 |
 | file://:0:0:0:0 | _Imaginary long double | 16 |

cpp/ql/test/library-tests/unspecified_type/types/unspecified_type.expected

@@ -15,7 +15,6 @@
 | file://:0:0:0:0 | _Float64 | _Float64 |
 | file://:0:0:0:0 | _Float64x | _Float64x |
 | file://:0:0:0:0 | _Float128 | _Float128 |
-| file://:0:0:0:0 | _Float128x | _Float128x |
 | file://:0:0:0:0 | _Imaginary double | _Imaginary double |
 | file://:0:0:0:0 | _Imaginary float | _Imaginary float |
 | file://:0:0:0:0 | _Imaginary long double | _Imaginary long double |

cpp/ql/test/library-tests/variables/variables/types.expected

@@ -14,7 +14,6 @@
 | _Float64 | BinaryFloatingPointType, RealNumberType |  |  |  |  |
 | _Float64x | BinaryFloatingPointType, RealNumberType |  |  |  |  |
 | _Float128 | BinaryFloatingPointType, RealNumberType |  |  |  |  |
-| _Float128x | BinaryFloatingPointType, RealNumberType |  |  |  |  |
 | _Imaginary double | BinaryFloatingPointType, ImaginaryNumberType |  |  |  |  |
 | _Imaginary float | BinaryFloatingPointType, ImaginaryNumberType |  |  |  |  |
 | _Imaginary long double | BinaryFloatingPointType, ImaginaryNumberType |  |  |  |  |

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Runtime.cs

@@ -22,7 +22,7 @@ namespace Semmle.Extraction.CSharp.Standalone
 
         public Runtime(IDotNet dotNet) => this.dotNet = dotNet;
 
-        internal sealed class RuntimeVersion : IComparable<RuntimeVersion>
+        internal record RuntimeVersion : IComparable<RuntimeVersion>
         {
             private readonly string dir;
             private readonly Version version;
@@ -71,11 +71,6 @@ namespace Semmle.Extraction.CSharp.Standalone
                 return c;
             }
 
-            public override bool Equals(object? obj) =>
-                obj is not null && obj is RuntimeVersion other && other.FullPath == FullPath;
-
-            public override int GetHashCode() => FullPath.GetHashCode();
-
             public override string ToString() => FullPath;
         }
 
@@ -97,7 +92,7 @@ namespace Semmle.Extraction.CSharp.Standalone
                 var match = RuntimeRegex().Match(r);
                 if (match.Success)
                 {
-                    runtimes.AddOrUpdate(match.Groups[1].Value, new RuntimeVersion(match.Groups[6].Value, match.Groups[2].Value, match.Groups[4].Value, match.Groups[5].Value));
+                    runtimes.AddOrUpdateToLatest(match.Groups[1].Value, new RuntimeVersion(match.Groups[6].Value, match.Groups[2].Value, match.Groups[4].Value, match.Groups[5].Value));
                 }
             });
 

csharp/extractor/Semmle.Util/DictionaryExtensions.cs

@@ -22,9 +22,9 @@ namespace Semmle.Util
 
         /// <summary>
         /// Adds a new value or replaces the existing value (if the new value is greater than the existing) 
-        /// in dictionary for the given key.
+        /// in this dictionary for the given key.
         /// </summary>
-        public static void AddOrUpdate<T1, T2>(this Dictionary<T1, T2> dict, T1 key, T2 value) where T1 : notnull where T2 : IComparable<T2>
+        public static void AddOrUpdateToLatest<T1, T2>(this Dictionary<T1, T2> dict, T1 key, T2 value) where T1 : notnull where T2 : IComparable<T2>
         {
             if (!dict.TryGetValue(key, out var existing) || existing.CompareTo(value) < 0)
             {

csharp/ql/lib/change-notes/2023-08-04-netcore-identity-hardcoded-cred.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The query library for `cs/hardcoded-credentials` now excludes benign properties such as `UserNameClaimType` and `AllowedUserNameCharacters` from `Microsoft.AspNetCore.Identity` options classes.

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll

@@ -1,3 +1,3 @@
 private import DataFlowImplSpecific
-private import codeql.dataflow.DataFlowImpl
+private import codeql.dataflow.internal.DataFlowImpl
 import MakeImpl<CsharpDataFlow>

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll

@@ -1,3 +1,3 @@
 private import DataFlowImplSpecific
-private import codeql.dataflow.DataFlowImplCommon
+private import codeql.dataflow.internal.DataFlowImplCommon
 import MakeImplCommon<CsharpDataFlow>

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplSpecific.qll

@@ -2,7 +2,7 @@
  * Provides C#-specific definitions for use in the data flow library.
  */
 
-private import codeql.dataflow.DataFlowParameter
+private import codeql.dataflow.DataFlow
 
 module Private {
   import DataFlowPrivate
@@ -13,7 +13,7 @@ module Public {
   import DataFlowPublic
 }
 
-module CsharpDataFlow implements DataFlowParameter {
+module CsharpDataFlow implements InputSig {
   import Private
   import Public
 

csharp/ql/lib/semmle/code/csharp/security/dataflow/HardcodedCredentialsQuery.qll

@@ -169,7 +169,7 @@ private class CredentialVar extends Assignable {
     exists(string name | name = this.getName() |
       name.regexpMatch("(?i).*pass(wd|word|code|phrase)(?!.*question).*")
       or
-      name.regexpMatch("(?i).*(puid|username|userid).*")
+      name.regexpMatch("(?i).*(puid|username|userid)(?!.*(characters|claimtype)).*")
       or
       name.regexpMatch("(?i).*(cert)(?!.*(format|name)).*")
     )

csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedConnectionString.expected

@@ -1,8 +1,8 @@
 edges
 nodes
-| HardcodedCredentials.cs:54:48:54:63 | "Password=12345" | semmle.label | "Password=12345" |
-| HardcodedCredentials.cs:56:49:56:63 | "User Id=12345" | semmle.label | "User Id=12345" |
+| HardcodedCredentials.cs:55:48:55:63 | "Password=12345" | semmle.label | "Password=12345" |
+| HardcodedCredentials.cs:57:49:57:63 | "User Id=12345" | semmle.label | "User Id=12345" |
 subpaths
 #select
-| HardcodedCredentials.cs:54:48:54:63 | "Password=12345" | HardcodedCredentials.cs:54:48:54:63 | "Password=12345" | HardcodedCredentials.cs:54:48:54:63 | "Password=12345" | 'ConnectionString' property includes hard-coded credentials set in $@. | HardcodedCredentials.cs:54:30:54:64 | object creation of type SqlConnection | object creation of type SqlConnection |
-| HardcodedCredentials.cs:56:49:56:63 | "User Id=12345" | HardcodedCredentials.cs:56:49:56:63 | "User Id=12345" | HardcodedCredentials.cs:56:49:56:63 | "User Id=12345" | 'ConnectionString' property includes hard-coded credentials set in $@. | HardcodedCredentials.cs:56:31:56:64 | object creation of type SqlConnection | object creation of type SqlConnection |
+| HardcodedCredentials.cs:55:48:55:63 | "Password=12345" | HardcodedCredentials.cs:55:48:55:63 | "Password=12345" | HardcodedCredentials.cs:55:48:55:63 | "Password=12345" | 'ConnectionString' property includes hard-coded credentials set in $@. | HardcodedCredentials.cs:55:30:55:64 | object creation of type SqlConnection | object creation of type SqlConnection |
+| HardcodedCredentials.cs:57:49:57:63 | "User Id=12345" | HardcodedCredentials.cs:57:49:57:63 | "User Id=12345" | HardcodedCredentials.cs:57:49:57:63 | "User Id=12345" | 'ConnectionString' property includes hard-coded credentials set in $@. | HardcodedCredentials.cs:57:31:57:64 | object creation of type SqlConnection | object creation of type SqlConnection |

csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedCredentials.cs

@@ -3,6 +3,7 @@ using System.Data.SqlClient;
 using System.Web;
 using System.Web.Security;
 using System.Security.Cryptography.X509Certificates;
+using Microsoft.AspNetCore.Identity;
 
 public class HardcodedHandler : IHttpHandler
 {
@@ -72,6 +73,21 @@ public class HardcodedHandler : IHttpHandler
 
         // BAD: Hard-coded user
         Membership.CreateUser("myusername", "mypassword");
+
+        var identityOptions = new IdentityOptions
+        {
+            User = new UserOptions
+            {
+                // GOOD: This is not a credential so hardcoding a string assignment is fine     
+                AllowedUserNameCharacters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._@+"
+            }
+        };
+
+        var claimsIdentityOptions = new ClaimsIdentityOptions
+        {
+            // GOOD: This is not a credential so hardcoding a string assignment is fine
+            UserNameClaimType = "username"
+        };
     }
 
     class Foo

csharp/ql/test/query-tests/Security Features/CWE-798/HardcodedCredentials.expected

@@ -1,24 +1,24 @@
 edges
-| HardcodedCredentials.cs:47:30:47:60 | array creation of type Byte[] : Byte[] | HardcodedCredentials.cs:50:13:50:23 | access to local variable rawCertData |
+| HardcodedCredentials.cs:48:30:48:60 | array creation of type Byte[] : Byte[] | HardcodedCredentials.cs:51:13:51:23 | access to local variable rawCertData |
 nodes
-| HardcodedCredentials.cs:15:25:15:36 | "myPa55word" | semmle.label | "myPa55word" |
-| HardcodedCredentials.cs:31:19:31:28 | "username" | semmle.label | "username" |
-| HardcodedCredentials.cs:45:39:45:53 | "myNewPa55word" | semmle.label | "myNewPa55word" |
-| HardcodedCredentials.cs:47:30:47:60 | array creation of type Byte[] : Byte[] | semmle.label | array creation of type Byte[] : Byte[] |
-| HardcodedCredentials.cs:50:13:50:23 | access to local variable rawCertData | semmle.label | access to local variable rawCertData |
-| HardcodedCredentials.cs:51:13:51:24 | "myPa55word" | semmle.label | "myPa55word" |
-| HardcodedCredentials.cs:74:31:74:42 | "myusername" | semmle.label | "myusername" |
-| HardcodedCredentials.cs:74:45:74:56 | "mypassword" | semmle.label | "mypassword" |
+| HardcodedCredentials.cs:16:25:16:36 | "myPa55word" | semmle.label | "myPa55word" |
+| HardcodedCredentials.cs:32:19:32:28 | "username" | semmle.label | "username" |
+| HardcodedCredentials.cs:46:39:46:53 | "myNewPa55word" | semmle.label | "myNewPa55word" |
+| HardcodedCredentials.cs:48:30:48:60 | array creation of type Byte[] : Byte[] | semmle.label | array creation of type Byte[] : Byte[] |
+| HardcodedCredentials.cs:51:13:51:23 | access to local variable rawCertData | semmle.label | access to local variable rawCertData |
+| HardcodedCredentials.cs:52:13:52:24 | "myPa55word" | semmle.label | "myPa55word" |
+| HardcodedCredentials.cs:75:31:75:42 | "myusername" | semmle.label | "myusername" |
+| HardcodedCredentials.cs:75:45:75:56 | "mypassword" | semmle.label | "mypassword" |
 | TestHardcodedCredentials.cs:21:31:21:42 | "myusername" | semmle.label | "myusername" |
 | TestHardcodedCredentials.cs:21:45:21:56 | "mypassword" | semmle.label | "mypassword" |
 | TestHardcodedCredentials.cs:26:19:26:28 | "username" | semmle.label | "username" |
 subpaths
 #select
-| HardcodedCredentials.cs:15:25:15:36 | "myPa55word" | HardcodedCredentials.cs:15:25:15:36 | "myPa55word" | HardcodedCredentials.cs:15:25:15:36 | "myPa55word" | The hard-coded value "myPa55word" flows to $@ which is compared against $@. | HardcodedCredentials.cs:15:25:15:36 | "myPa55word" | "myPa55word" | HardcodedCredentials.cs:15:13:15:20 | access to local variable password | access to local variable password |
-| HardcodedCredentials.cs:31:19:31:28 | "username" | HardcodedCredentials.cs:31:19:31:28 | "username" | HardcodedCredentials.cs:31:19:31:28 | "username" | The hard-coded value "username" flows to the $@ parameter in $@. | HardcodedCredentials.cs:31:19:31:28 | "username" | name | HardcodedCredentials.cs:29:31:43:13 | object creation of type MembershipUser | object creation of type MembershipUser |
-| HardcodedCredentials.cs:45:39:45:53 | "myNewPa55word" | HardcodedCredentials.cs:45:39:45:53 | "myNewPa55word" | HardcodedCredentials.cs:45:39:45:53 | "myNewPa55word" | The hard-coded value "myNewPa55word" flows to the $@ parameter in $@. | HardcodedCredentials.cs:45:39:45:53 | "myNewPa55word" | newPassword | HardcodedCredentials.cs:45:9:45:54 | call to method ChangePassword | call to method ChangePassword |
-| HardcodedCredentials.cs:47:30:47:60 | array creation of type Byte[] | HardcodedCredentials.cs:47:30:47:60 | array creation of type Byte[] : Byte[] | HardcodedCredentials.cs:50:13:50:23 | access to local variable rawCertData | This hard-coded value flows to the $@ parameter in $@. | HardcodedCredentials.cs:50:13:50:23 | access to local variable rawCertData | rawData | HardcodedCredentials.cs:49:33:51:25 | object creation of type X509Certificate2 | object creation of type X509Certificate2 |
-| HardcodedCredentials.cs:51:13:51:24 | "myPa55word" | HardcodedCredentials.cs:51:13:51:24 | "myPa55word" | HardcodedCredentials.cs:51:13:51:24 | "myPa55word" | The hard-coded value "myPa55word" flows to the $@ parameter in $@. | HardcodedCredentials.cs:51:13:51:24 | "myPa55word" | password | HardcodedCredentials.cs:49:33:51:25 | object creation of type X509Certificate2 | object creation of type X509Certificate2 |
-| HardcodedCredentials.cs:74:31:74:42 | "myusername" | HardcodedCredentials.cs:74:31:74:42 | "myusername" | HardcodedCredentials.cs:74:31:74:42 | "myusername" | The hard-coded value "myusername" flows to the $@ parameter in $@. | HardcodedCredentials.cs:74:31:74:42 | "myusername" | username | HardcodedCredentials.cs:74:9:74:57 | call to method CreateUser | call to method CreateUser |
-| HardcodedCredentials.cs:74:45:74:56 | "mypassword" | HardcodedCredentials.cs:74:45:74:56 | "mypassword" | HardcodedCredentials.cs:74:45:74:56 | "mypassword" | The hard-coded value "mypassword" flows to the $@ parameter in $@. | HardcodedCredentials.cs:74:45:74:56 | "mypassword" | password | HardcodedCredentials.cs:74:9:74:57 | call to method CreateUser | call to method CreateUser |
+| HardcodedCredentials.cs:16:25:16:36 | "myPa55word" | HardcodedCredentials.cs:16:25:16:36 | "myPa55word" | HardcodedCredentials.cs:16:25:16:36 | "myPa55word" | The hard-coded value "myPa55word" flows to $@ which is compared against $@. | HardcodedCredentials.cs:16:25:16:36 | "myPa55word" | "myPa55word" | HardcodedCredentials.cs:16:13:16:20 | access to local variable password | access to local variable password |
+| HardcodedCredentials.cs:32:19:32:28 | "username" | HardcodedCredentials.cs:32:19:32:28 | "username" | HardcodedCredentials.cs:32:19:32:28 | "username" | The hard-coded value "username" flows to the $@ parameter in $@. | HardcodedCredentials.cs:32:19:32:28 | "username" | name | HardcodedCredentials.cs:30:31:44:13 | object creation of type MembershipUser | object creation of type MembershipUser |
+| HardcodedCredentials.cs:46:39:46:53 | "myNewPa55word" | HardcodedCredentials.cs:46:39:46:53 | "myNewPa55word" | HardcodedCredentials.cs:46:39:46:53 | "myNewPa55word" | The hard-coded value "myNewPa55word" flows to the $@ parameter in $@. | HardcodedCredentials.cs:46:39:46:53 | "myNewPa55word" | newPassword | HardcodedCredentials.cs:46:9:46:54 | call to method ChangePassword | call to method ChangePassword |
+| HardcodedCredentials.cs:48:30:48:60 | array creation of type Byte[] | HardcodedCredentials.cs:48:30:48:60 | array creation of type Byte[] : Byte[] | HardcodedCredentials.cs:51:13:51:23 | access to local variable rawCertData | This hard-coded value flows to the $@ parameter in $@. | HardcodedCredentials.cs:51:13:51:23 | access to local variable rawCertData | rawData | HardcodedCredentials.cs:50:33:52:25 | object creation of type X509Certificate2 | object creation of type X509Certificate2 |
+| HardcodedCredentials.cs:52:13:52:24 | "myPa55word" | HardcodedCredentials.cs:52:13:52:24 | "myPa55word" | HardcodedCredentials.cs:52:13:52:24 | "myPa55word" | The hard-coded value "myPa55word" flows to the $@ parameter in $@. | HardcodedCredentials.cs:52:13:52:24 | "myPa55word" | password | HardcodedCredentials.cs:50:33:52:25 | object creation of type X509Certificate2 | object creation of type X509Certificate2 |
+| HardcodedCredentials.cs:75:31:75:42 | "myusername" | HardcodedCredentials.cs:75:31:75:42 | "myusername" | HardcodedCredentials.cs:75:31:75:42 | "myusername" | The hard-coded value "myusername" flows to the $@ parameter in $@. | HardcodedCredentials.cs:75:31:75:42 | "myusername" | username | HardcodedCredentials.cs:75:9:75:57 | call to method CreateUser | call to method CreateUser |
+| HardcodedCredentials.cs:75:45:75:56 | "mypassword" | HardcodedCredentials.cs:75:45:75:56 | "mypassword" | HardcodedCredentials.cs:75:45:75:56 | "mypassword" | The hard-coded value "mypassword" flows to the $@ parameter in $@. | HardcodedCredentials.cs:75:45:75:56 | "mypassword" | password | HardcodedCredentials.cs:75:9:75:57 | call to method CreateUser | call to method CreateUser |
 | TestHardcodedCredentials.cs:26:19:26:28 | "username" | TestHardcodedCredentials.cs:26:19:26:28 | "username" | TestHardcodedCredentials.cs:26:19:26:28 | "username" | The hard-coded value "username" flows to the $@ parameter in $@. | TestHardcodedCredentials.cs:26:19:26:28 | "username" | name | TestHardcodedCredentials.cs:24:31:38:13 | object creation of type MembershipUser | object creation of type MembershipUser |

csharp/ql/test/query-tests/Security Features/CWE-798/options

@@ -1,4 +1,5 @@
 semmle-extractor-options: /nostdlib /noconfig
 semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/System.Data.SqlClient/4.8.3/System.Data.SqlClient.csproj
 semmle-extractor-options: ${testdir}/../../../resources/stubs/System.Web.cs
+semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/_frameworks/Microsoft.AspNetCore.App/Microsoft.AspNetCore.App.csproj
 semmle-extractor-options: ${testdir}/../../../resources/stubs/Microsoft.VisualStudio.TestTools.UnitTesting.cs

go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll

@@ -1,3 +1,3 @@
 private import DataFlowImplSpecific
-private import codeql.dataflow.DataFlowImpl
+private import codeql.dataflow.internal.DataFlowImpl
 import MakeImpl<GoDataFlow>

go/ql/lib/semmle/go/dataflow/internal/DataFlowImplCommon.qll

@@ -1,3 +1,3 @@
 private import DataFlowImplSpecific
-private import codeql.dataflow.DataFlowImplCommon
+private import codeql.dataflow.internal.DataFlowImplCommon
 import MakeImplCommon<GoDataFlow>

go/ql/lib/semmle/go/dataflow/internal/DataFlowImplSpecific.qll

@@ -2,7 +2,7 @@
  * Provides Go-specific definitions for use in the data flow library.
  */
 
-private import codeql.dataflow.DataFlowParameter
+private import codeql.dataflow.DataFlow
 
 module Private {
   import DataFlowPrivate
@@ -13,7 +13,7 @@ module Public {
   import DataFlowUtil
 }
 
-module GoDataFlow implements DataFlowParameter {
+module GoDataFlow implements InputSig {
   import Private
   import Public
 

java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt

@@ -552,7 +552,7 @@ open class KotlinFileExtractor(
                 logger.warnElement("Expected annotation property to define a getter", prop)
             } else {
                 val getterId = useFunction<DbMethod>(getter)
-                val exprId = extractAnnotationValueExpression(v, id, i, "{${getterId}}", getter.returnType, extractEnumTypeAccesses)
+                val exprId = extractAnnotationValueExpression(v, id, i, "{$getterId}", getter.returnType, extractEnumTypeAccesses)
                 if (exprId != null) {
                     tw.writeAnnotValue(id, getterId, exprId)
                 }
@@ -587,7 +587,7 @@ open class KotlinFileExtractor(
                 extractAnnotation(v, parent, idx, extractEnumTypeAccesses, contextLabel)
             }
             is IrVararg -> {
-                tw.getLabelFor<DbArrayinit>("@\"annotationarray;{${parent}};$contextLabel\"").also { arrayId ->
+                tw.getLabelFor<DbArrayinit>("@\"annotationarray;{$parent};$contextLabel\"").also { arrayId ->
                     // Use the context type (i.e., the type the annotation expects, not the actual type of the array)
                     // because the Java extractor fills in array types using the same technique. These should only
                     // differ for generic annotations.
@@ -1193,7 +1193,7 @@ open class KotlinFileExtractor(
             // n + o'th parameter, where `o` is the parameter offset caused by adding any dispatch receiver to the parameter list.
             // Note we don't need to add the extension receiver here because `useValueParameter` always assumes an extension receiver
             // will be prepended if one exists.
-            val realFunctionId = useFunction<DbCallable>(f)
+            val realFunctionId = useFunction<DbCallable>(f, parentId, null)
             DeclarationStackAdjuster(f, OverriddenFunctionAttributes(id, id, locId, nonSyntheticParams, typeParameters = listOf(), isStatic = true)).use {
                 val realParamsVarId = getValueParameterLabel(id, parameterTypes.size - 2)
                 val intType = pluginContext.irBuiltIns.intType

java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt

@@ -612,7 +612,7 @@ open class KotlinUsesExtractor(
         val componentTypeLabel = recInfo.componentTypeResults.javaResult.id
         val dimensions = recInfo.dimensions + 1
 
-        val id = tw.getLabelFor<DbArray>("@\"array;$dimensions;{${elementTypeLabel}}\"") {
+        val id = tw.getLabelFor<DbArray>("@\"array;$dimensions;{$elementTypeLabel}\"") {
             tw.writeArrays(
                 it,
                 javaShortName,
@@ -1141,7 +1141,7 @@ open class KotlinUsesExtractor(
         // method (and presumably that disambiguation is never needed when the method belongs to a parameterized
         // instance of a generic class), but as of now I don't know when the raw method would be referred to.
         val typeArgSuffix = if (functionTypeParameters.isNotEmpty() && classTypeArgsIncludingOuterClasses.isNullOrEmpty()) "<${functionTypeParameters.size}>" else "";
-        return "@\"$prefix;{$parentId}.$name($paramTypeIds){$returnTypeId}${typeArgSuffix}\""
+        return "@\"$prefix;{$parentId}.$name($paramTypeIds){$returnTypeId}$typeArgSuffix\""
     }
 
     val javaLangClass by lazy { referenceExternalClass("java.lang.Class") }
@@ -1672,7 +1672,7 @@ open class KotlinUsesExtractor(
         // clashing trap labels. These are always private, so we can just make up a label without
         // worrying about their names as seen from Java.
         val extensionPropertyDiscriminator = getExtensionReceiverType(f)?.let { "extension;${useType(it).javaResult.id}" } ?: ""
-        return "@\"field;{$parentId};${extensionPropertyDiscriminator}${f.name.asString()}\""
+        return "@\"field;{$parentId};$extensionPropertyDiscriminator${f.name.asString()}\""
     }
 
     fun useField(f: IrField): Label<out DbField> =

java/ql/lib/change-notes/2023-06-16-initial-version.md

@@ -0,0 +1,4 @@
+---
+category: majorAnalysis
+---
+* Improved support for flow through captured variables that properly adheres to inter-procedural control flow.

java/ql/lib/change-notes/2023-08-07-randomdatasource-typo-fix.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Fixed a typo in the `StdlibRandomSource` class in `RandomDataSource.qll`, which caused the class to improperly model calls to the `nextBytes` method. Queries relying on `StdlibRandomSource` may see an increase in results.

java/ql/lib/ext/java.lang.model.yml

@@ -176,7 +176,6 @@ extensions:
       - ["java.lang", "Object", "getClass", "()", "summary", "manual"]
       - ["java.lang", "Object", "hashCode", "()", "summary", "manual"]
       - ["java.lang", "Object", "toString", "()", "summary", "manual"]
-      - ["java.lang", "Runnable", "run", "()", "summary", "manual"]
       - ["java.lang", "Runtime", "getRuntime", "()", "summary", "manual"]
       - ["java.lang", "String", "compareTo", "(String)", "summary", "manual"]
       - ["java.lang", "String", "contains", "(CharSequence)", "summary", "manual"]

java/ql/lib/ext/java.nio.file.model.yml

@@ -17,11 +17,11 @@ extensions:
       - ["java.nio.file", "Files", False, "createTempFile", "(Path,String,String,FileAttribute[])", "", "Argument[0]", "path-injection", "manual"]
       - ["java.nio.file", "Files", False, "delete", "(Path)", "", "Argument[0]", "path-injection", "ai-manual"]
       - ["java.nio.file", "Files", False, "deleteIfExists", "(Path)", "", "Argument[0]", "path-injection", "ai-manual"]
-      - ["java.nio.file", "Files", False, "deleteIfExists", "(Path)", "", "Argument[0]", "path-injection", "ai-manual"]
       - ["java.nio.file", "Files", False, "getFileStore", "(Path)", "", "Argument[0]", "path-injection", "ai-manual"] # the FileStore class is unlikely to be used for later sanitization
       - ["java.nio.file", "Files", False, "lines", "(Path,Charset)", "", "Argument[0]", "path-injection", "ai-manual"]
       - ["java.nio.file", "Files", False, "lines", "(Path)", "", "Argument[0]", "path-injection", "ai-manual"]
       - ["java.nio.file", "Files", False, "move", "", "", "Argument[1]", "path-injection", "manual"]
+      - ["java.nio.file", "Files", False, "move", "(Path,Path,CopyOption[])", "", "Argument[0]", "path-injection", "ai-manual"]
       - ["java.nio.file", "Files", False, "newBufferedReader", "(Path,Charset)", "", "Argument[0]", "path-injection", "ai-manual"]
       - ["java.nio.file", "Files", False, "newBufferedReader", "(Path)", "", "Argument[0]", "path-injection", "ai-manual"]
       - ["java.nio.file", "Files", False, "newBufferedWriter", "", "", "Argument[0]", "path-injection", "manual"]
@@ -37,11 +37,6 @@ extensions:
       - ["java.nio.file", "Files", False, "write", "", "", "Argument[1]", "file-content-store", "manual"]
       - ["java.nio.file", "Files", False, "writeString", "", "", "Argument[0]", "path-injection", "manual"]
       - ["java.nio.file", "Files", False, "writeString", "", "", "Argument[1]", "file-content-store", "manual"]
-      - ["java.nio.file", "Files", True, "move", "(Path,Path,CopyOption[])", "", "Argument[1]", "path-injection", "ai-manual"]
-      - ["java.nio.file", "Files", True, "move", "(Path,Path,CopyOption[])", "", "Argument[0]", "path-injection", "ai-manual"]
-      - ["java.nio.file", "Files", True, "delete", "(Path)", "", "Argument[0]", "path-injection", "ai-manual"]
-      - ["java.nio.file", "Files", True, "newInputStream", "(Path,OpenOption[])", "", "Argument[0]", "path-injection", "ai-manual"]
-      - ["java.nio.file", "Files", True, "newOutputStream", "(Path,OpenOption[])", "", "Argument[0]", "path-injection", "ai-manual"]
       - ["java.nio.file", "FileSystem", False, "getPath", "", "", "Argument[0..1]", "path-injection", "manual"] # old PathCreation
       - ["java.nio.file", "FileSystems", False, "newFileSystem", "(URI,Map)", "", "Argument[0]", "path-injection", "ai-manual"]
       - ["java.nio.file", "FileSystems", False, "newFileSystem", "(URI,Map)", "", "Argument[0]", "request-forgery", "ai-manual"]

java/ql/lib/ext/threatmodels/supported-threat-models.model.yml

@@ -0,0 +1,7 @@
+extensions:
+
+  - addsTo:
+      pack: codeql/java-all
+      extensible: supportedThreatModels
+    data:
+      - ["default"] # The "default" threat model is always included.

java/ql/lib/ext/threatmodels/threat-model-grouping.model.yml

@@ -0,0 +1,23 @@
+extensions:
+
+  - addsTo:
+      pack: codeql/java-all
+      extensible: threatModelGrouping
+    data:
+      # Default threat model
+      - ["remote", "default"]
+      - ["uri-path", "default"]
+
+      # Android threat models
+      - ["android-external-storage-dir", "android"]
+      - ["contentprovider", "android"]
+
+      # Remote threat models
+      - ["request", "remote"]
+      - ["response", "remote"]
+
+      # Local threat models
+      - ["database", "local"]
+      - ["cli", "local"]
+      - ["environment", "local"]
+      - ["file", "local"]

java/ql/lib/qlpack.yml

@@ -16,4 +16,5 @@ dataExtensions:
   - ext/*.model.yml
   - ext/generated/*.model.yml
   - ext/experimental/*.model.yml
+  - ext/threatmodels/*.model.yml
 warnOnImplicitThis: true

java/ql/lib/semmle/code/java/dataflow/ExternalFlowConfiguration.qll

@@ -0,0 +1,31 @@
+/**
+ * INTERNAL use only. This is an experimental API subject to change without notice.
+ *
+ * This module provides extensible predicates for configuring which kinds of MaD models
+ * are applicable to generic queries.
+ */
+
+private import ExternalFlowExtensions
+
+/**
+ * Holds if the specified kind of source model is supported for the current query.
+ */
+extensible private predicate supportedThreatModels(string kind);
+
+/**
+ * Holds if the specified kind of source model is containted within the specified group.
+ */
+extensible private predicate threatModelGrouping(string kind, string group);
+
+/**
+ * Gets the threat models that are direct descendants of the specified kind/group.
+ */
+private string getChildThreatModel(string group) { threatModelGrouping(result, group) }
+
+/**
+ * Holds if the source model kind `kind` is relevant for generic queries
+ * under the current threat model configuration.
+ */
+predicate sourceModelKindConfig(string kind) {
+  exists(string group | supportedThreatModels(group) and kind = getChildThreatModel*(group))
+}

java/ql/lib/semmle/code/java/dataflow/FlowSummary.qll

@@ -101,6 +101,7 @@ abstract class SyntheticCallable extends string {
  * A module for importing frameworks that define synthetic callables.
  */
 private module SyntheticCallables {
+  private import semmle.code.java.dispatch.WrappedInvocation
   private import semmle.code.java.frameworks.android.Intent
   private import semmle.code.java.frameworks.Stream
 }
@@ -170,6 +171,8 @@ class SummarizedCallableBase extends TSummarizedCallableBase {
   }
 }
 
+class Provenance = Impl::Public::Provenance;
+
 class SummarizedCallable = Impl::Public::SummarizedCallable;
 
 class NeutralCallable = Impl::Public::NeutralCallable;

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll

@@ -1,3 +1,3 @@
 private import DataFlowImplSpecific
-private import codeql.dataflow.DataFlowImpl
+private import codeql.dataflow.internal.DataFlowImpl
 import MakeImpl<JavaDataFlow>

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll

@@ -1,3 +1,3 @@
 private import DataFlowImplSpecific
-private import codeql.dataflow.DataFlowImplCommon
+private import codeql.dataflow.internal.DataFlowImplCommon
 import MakeImplCommon<JavaDataFlow>

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplSpecific.qll

@@ -2,7 +2,7 @@
  * Provides Java-specific definitions for use in the data flow library.
  */
 
-private import codeql.dataflow.DataFlowParameter
+private import codeql.dataflow.DataFlow
 
 module Private {
   import DataFlowPrivate
@@ -13,7 +13,7 @@ module Public {
   import DataFlowUtil
 }
 
-module JavaDataFlow implements DataFlowParameter {
+module JavaDataFlow implements InputSig {
   import Private
   import Public
 

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowNodes.qll

@@ -55,7 +55,8 @@ private module Cached {
       )
     } or
     TFlowSummaryNode(FlowSummaryImpl::Private::SummaryNode sn) or
-    TFieldValueNode(Field f)
+    TFieldValueNode(Field f) or
+    TCaptureNode(CaptureFlow::SynthesizedCaptureNode cn)
 
   cached
   newtype TContent =
@@ -64,6 +65,7 @@ private module Cached {
     TCollectionContent() or
     TMapKeyContent() or
     TMapValueContent() or
+    TCapturedVariableContent(CapturedVariable v) or
     TSyntheticFieldContent(SyntheticField s)
 
   cached
@@ -73,6 +75,7 @@ private module Cached {
     TCollectionContentApprox() or
     TMapKeyContentApprox() or
     TMapValueContentApprox() or
+    TCapturedVariableContentApprox(CapturedVariable v) or
     TSyntheticFieldApproxContent()
 }
 
@@ -127,6 +130,8 @@ module Public {
       or
       result = this.(ImplicitPostUpdateNode).getPreUpdateNode().getType()
       or
+      result = this.(CaptureNode).getTypeImpl()
+      or
       result = this.(FieldValueNode).getField().getType()
     }
 
@@ -372,6 +377,7 @@ module Private {
     result.asCallable() = n.(MallocNode).getClassInstanceExpr().getEnclosingCallable() or
     result = nodeGetEnclosingCallable(n.(ImplicitPostUpdateNode).getPreUpdateNode()) or
     result.asSummarizedCallable() = n.(FlowSummaryNode).getSummarizedCallable() or
+    result.asCallable() = n.(CaptureNode).getSynthesizedCaptureNode().getEnclosingCallable() or
     result.asFieldScope() = n.(FieldValueNode).getField()
   }
 
@@ -491,6 +497,28 @@ module Private {
       c.asSummarizedCallable() = this.getSummarizedCallable() and pos = this.getPosition()
     }
   }
+
+  /**
+   * A synthesized data flow node representing a closure object that tracks
+   * captured variables.
+   */
+  class CaptureNode extends Node, TCaptureNode {
+    private CaptureFlow::SynthesizedCaptureNode cn;
+
+    CaptureNode() { this = TCaptureNode(cn) }
+
+    CaptureFlow::SynthesizedCaptureNode getSynthesizedCaptureNode() { result = cn }
+
+    override Location getLocation() { result = cn.getLocation() }
+
+    override string toString() { result = cn.toString() }
+
+    Type getTypeImpl() {
+      exists(Variable v | cn.isVariableAccess(v) and result = v.getType())
+      or
+      cn.isInstanceAccess() and result = cn.getEnclosingCallable().getDeclaringType()
+    }
+  }
 }
 
 private import Private
@@ -520,3 +548,14 @@ private class SummaryPostUpdateNode extends FlowSummaryNode, PostUpdateNode {
 
   override Node getPreUpdateNode() { result = pre }
 }
+
+private class CapturePostUpdateNode extends PostUpdateNode, CaptureNode {
+  private CaptureNode pre;
+
+  CapturePostUpdateNode() {
+    CaptureFlow::capturePostUpdateNode(this.getSynthesizedCaptureNode(),
+      pre.getSynthesizedCaptureNode())
+  }
+
+  override Node getPreUpdateNode() { result = pre }
+}

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowPrivate.qll

@@ -10,6 +10,7 @@ private import semmle.code.java.dataflow.FlowSummary
 private import FlowSummaryImpl as FlowSummaryImpl
 private import DataFlowImplConsistency
 private import DataFlowNodes
+private import codeql.dataflow.VariableCapture as VariableCapture
 import DataFlowNodes::Private
 
 private newtype TReturnKind = TNormalReturnKind()
@@ -51,26 +52,131 @@ private predicate fieldStep(Node node1, Node node2) {
   )
 }
 
-/**
- * Holds if data can flow from `node1` to `node2` through variable capture.
- */
-private predicate variableCaptureStep(Node node1, ExprNode node2) {
-  exists(SsaImplicitInit closure, SsaVariable captured |
-    closure.captures(captured) and
-    node2.getExpr() = closure.getAFirstUse()
-  |
-    node1.asExpr() = captured.getAUse()
-    or
-    not exists(captured.getAUse()) and
-    exists(SsaVariable capturedDef | capturedDef = captured.getAnUltimateDefinition() |
-      capturedDef.(SsaImplicitInit).isParameterDefinition(node1.asParameter()) or
-      capturedDef.(SsaExplicitUpdate).getDefiningExpr().(VariableAssign).getSource() =
-        node1.asExpr() or
-      capturedDef.(SsaExplicitUpdate).getDefiningExpr().(AssignOp) = node1.asExpr()
-    )
+private predicate closureFlowStep(Expr e1, Expr e2) {
+  simpleAstFlowStep(e1, e2)
+  or
+  exists(SsaVariable v |
+    v.getAUse() = e2 and
+    v.getAnUltimateDefinition().(SsaExplicitUpdate).getDefiningExpr().(VariableAssign).getSource() =
+      e1
   )
 }
 
+private module CaptureInput implements VariableCapture::InputSig {
+  private import java as J
+
+  class Location = J::Location;
+
+  class BasicBlock instanceof J::BasicBlock {
+    string toString() { result = super.toString() }
+
+    Callable getEnclosingCallable() { result = super.getEnclosingCallable() }
+
+    Location getLocation() { result = super.getLocation() }
+  }
+
+  BasicBlock getImmediateBasicBlockDominator(BasicBlock bb) { bbIDominates(result, bb) }
+
+  BasicBlock getABasicBlockSuccessor(BasicBlock bb) {
+    result = bb.(J::BasicBlock).getABBSuccessor()
+  }
+
+  //TODO: support capture of `this` in lambdas
+  class CapturedVariable instanceof LocalScopeVariable {
+    CapturedVariable() {
+      2 <=
+        strictcount(J::Callable c |
+          c = this.getCallable() or c = this.getAnAccess().getEnclosingCallable()
+        )
+    }
+
+    string toString() { result = super.toString() }
+
+    Callable getCallable() { result = super.getCallable() }
+
+    Location getLocation() { result = super.getLocation() }
+  }
+
+  class CapturedParameter extends CapturedVariable instanceof Parameter { }
+
+  class Expr instanceof J::Expr {
+    string toString() { result = super.toString() }
+
+    Location getLocation() { result = super.getLocation() }
+
+    predicate hasCfgNode(BasicBlock bb, int i) { this = bb.(J::BasicBlock).getNode(i) }
+  }
+
+  class VariableWrite extends Expr instanceof VariableUpdate {
+    CapturedVariable v;
+
+    VariableWrite() { super.getDestVar() = v }
+
+    CapturedVariable getVariable() { result = v }
+
+    Expr getSource() {
+      result = this.(VariableAssign).getSource() or
+      result = this.(AssignOp)
+    }
+  }
+
+  class VariableRead extends Expr instanceof RValue {
+    CapturedVariable v;
+
+    VariableRead() { super.getVariable() = v }
+
+    CapturedVariable getVariable() { result = v }
+  }
+
+  class ClosureExpr extends Expr instanceof ClassInstanceExpr {
+    NestedClass nc;
+
+    ClosureExpr() {
+      nc.(AnonymousClass).getClassInstanceExpr() = this
+      or
+      nc instanceof LocalClass and
+      super.getConstructedType().getASourceSupertype*().getSourceDeclaration() = nc
+    }
+
+    predicate hasBody(Callable body) { nc.getACallable() = body }
+
+    predicate hasAliasedAccess(Expr f) { closureFlowStep+(this, f) and not closureFlowStep(f, _) }
+  }
+
+  class Callable extends J::Callable {
+    predicate isConstructor() { this instanceof Constructor }
+  }
+}
+
+class CapturedVariable = CaptureInput::CapturedVariable;
+
+class CapturedParameter = CaptureInput::CapturedParameter;
+
+module CaptureFlow = VariableCapture::Flow<CaptureInput>;
+
+private CaptureFlow::ClosureNode asClosureNode(Node n) {
+  result = n.(CaptureNode).getSynthesizedCaptureNode() or
+  result.(CaptureFlow::ExprNode).getExpr() = n.asExpr() or
+  result.(CaptureFlow::ExprPostUpdateNode).getExpr() =
+    n.(PostUpdateNode).getPreUpdateNode().asExpr() or
+  result.(CaptureFlow::ParameterNode).getParameter() = n.asParameter() or
+  result.(CaptureFlow::ThisParameterNode).getCallable() = n.(InstanceParameterNode).getCallable() or
+  exprNode(result.(CaptureFlow::MallocNode).getClosureExpr()).(PostUpdateNode).getPreUpdateNode() =
+    n
+}
+
+private predicate captureStoreStep(Node node1, CapturedVariableContent c, Node node2) {
+  CaptureFlow::storeStep(asClosureNode(node1), c.getVariable(), asClosureNode(node2))
+}
+
+private predicate captureReadStep(Node node1, CapturedVariableContent c, Node node2) {
+  CaptureFlow::readStep(asClosureNode(node1), c.getVariable(), asClosureNode(node2))
+}
+
+predicate captureValueStep(Node node1, Node node2) {
+  CaptureFlow::localFlowStep(asClosureNode(node1), asClosureNode(node2))
+}
+
 /**
  * Holds if data can flow from `node1` to `node2` through a field or
  * variable capture.
@@ -78,10 +184,6 @@ private predicate variableCaptureStep(Node node1, ExprNode node2) {
 predicate jumpStep(Node node1, Node node2) {
   fieldStep(node1, node2)
   or
-  variableCaptureStep(node1, node2)
-  or
-  variableCaptureStep(node1.(PostUpdateNode).getPreUpdateNode(), node2)
-  or
   any(AdditionalValueStep a).step(node1, node2) and
   node1.getEnclosingCallable() != node2.getEnclosingCallable()
   or
@@ -117,6 +219,8 @@ predicate storeStep(Node node1, ContentSet f, Node node2) {
   or
   FlowSummaryImpl::Private::Steps::summaryStoreStep(node1.(FlowSummaryNode).getSummaryNode(), f,
     node2.(FlowSummaryNode).getSummaryNode())
+  or
+  captureStoreStep(node1, f, node2)
 }
 
 /**
@@ -149,6 +253,8 @@ predicate readStep(Node node1, ContentSet f, Node node2) {
   or
   FlowSummaryImpl::Private::Steps::summaryReadStep(node1.(FlowSummaryNode).getSummaryNode(), f,
     node2.(FlowSummaryNode).getSummaryNode())
+  or
+  captureReadStep(node1, f, node2)
 }
 
 /**
@@ -231,19 +337,29 @@ private newtype TDataFlowCallable =
   TSummarizedCallable(SummarizedCallable c) or
   TFieldScope(Field f)
 
+/**
+ * A callable or scope enclosing some number of data flow nodes. This can either
+ * be a source callable, a synthesized callable for which we have a summary
+ * model, or a synthetic scope for a field value node.
+ */
 class DataFlowCallable extends TDataFlowCallable {
+  /** Gets the source callable corresponding to this callable, if any. */
   Callable asCallable() { this = TSrcCallable(result) }
 
+  /** Gets the summary model callable corresponding to this callable, if any. */
   SummarizedCallable asSummarizedCallable() { this = TSummarizedCallable(result) }
 
+  /** Gets the field corresponding to this callable, if it is a field value scope. */
   Field asFieldScope() { this = TFieldScope(result) }
 
+  /** Gets a textual representation of this callable. */
   string toString() {
     result = this.asCallable().toString() or
     result = "Synthetic: " + this.asSummarizedCallable().toString() or
     result = "Field scope: " + this.asFieldScope().toString()
   }
 
+  /** Gets the location of this callable. */
   Location getLocation() {
     result = this.asCallable().getLocation() or
     result = this.asSummarizedCallable().getLocation() or
@@ -406,6 +522,8 @@ predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preserves
  */
 predicate allowParameterReturnInSelf(ParameterNode p) {
   FlowSummaryImpl::Private::summaryAllowParameterReturnInSelf(p)
+  or
+  CaptureFlow::heuristicAllowInstanceParameterReturnInSelf(p.(InstanceParameterNode).getCallable())
 }
 
 /** An approximated `Content`. */
@@ -447,6 +565,10 @@ ContentApprox getContentApprox(Content c) {
   or
   c instanceof MapValueContent and result = TMapValueContentApprox()
   or
+  exists(CapturedVariable v |
+    c = TCapturedVariableContent(v) and result = TCapturedVariableContentApprox(v)
+  )
+  or
   c instanceof SyntheticFieldContent and result = TSyntheticFieldApproxContent()
 }
 

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowUtil.qll

@@ -135,6 +135,30 @@ private module Cached {
 
 import Cached
 
+private predicate capturedVariableRead(Node n) {
+  n.asExpr().(RValue).getVariable() instanceof CapturedVariable
+}
+
+/**
+ * Holds if there is a data flow step from `e1` to `e2` that only steps from
+ * child to parent in the AST.
+ */
+predicate simpleAstFlowStep(Expr e1, Expr e2) {
+  e2.(CastingExpr).getExpr() = e1
+  or
+  e2.(ChooseExpr).getAResultExpr() = e1
+  or
+  e2.(AssignExpr).getSource() = e1
+  or
+  e2.(ArrayCreationExpr).getInit() = e1
+  or
+  e2 = any(StmtExpr stmtExpr | e1 = stmtExpr.getResultExpr())
+  or
+  e2 = any(NotNullExpr nne | e1 = nne.getExpr())
+  or
+  e2.(WhenExpr).getBranch(_).getAResult() = e1
+}
+
 private predicate simpleLocalFlowStep0(Node node1, Node node2) {
   TaintTrackingUtil::forceCachingInSameStage() and
   // Variable flow steps through adjacent def-use and use-use pairs.
@@ -142,39 +166,31 @@ private predicate simpleLocalFlowStep0(Node node1, Node node2) {
     upd.getDefiningExpr().(VariableAssign).getSource() = node1.asExpr() or
     upd.getDefiningExpr().(AssignOp) = node1.asExpr()
   |
-    node2.asExpr() = upd.getAFirstUse()
+    node2.asExpr() = upd.getAFirstUse() and
+    not capturedVariableRead(node2)
   )
   or
   exists(SsaImplicitInit init |
     init.isParameterDefinition(node1.asParameter()) and
-    node2.asExpr() = init.getAFirstUse()
+    node2.asExpr() = init.getAFirstUse() and
+    not capturedVariableRead(node2)
   )
   or
   adjacentUseUse(node1.asExpr(), node2.asExpr()) and
   not exists(FieldRead fr |
     hasNonlocalValue(fr) and fr.getField().isStatic() and fr = node1.asExpr()
   ) and
-  not FlowSummaryImpl::Private::Steps::prohibitsUseUseFlow(node1, _)
+  not FlowSummaryImpl::Private::Steps::prohibitsUseUseFlow(node1, _) and
+  not capturedVariableRead(node2)
   or
   ThisFlow::adjacentThisRefs(node1, node2)
   or
-  adjacentUseUse(node1.(PostUpdateNode).getPreUpdateNode().asExpr(), node2.asExpr())
+  adjacentUseUse(node1.(PostUpdateNode).getPreUpdateNode().asExpr(), node2.asExpr()) and
+  not capturedVariableRead(node2)
   or
   ThisFlow::adjacentThisRefs(node1.(PostUpdateNode).getPreUpdateNode(), node2)
   or
-  node2.asExpr().(CastingExpr).getExpr() = node1.asExpr()
-  or
-  node2.asExpr().(ChooseExpr).getAResultExpr() = node1.asExpr()
-  or
-  node2.asExpr().(AssignExpr).getSource() = node1.asExpr()
-  or
-  node2.asExpr().(ArrayCreationExpr).getInit() = node1.asExpr()
-  or
-  node2.asExpr() = any(StmtExpr stmtExpr | node1.asExpr() = stmtExpr.getResultExpr())
-  or
-  node2.asExpr() = any(NotNullExpr nne | node1.asExpr() = nne.getExpr())
-  or
-  node2.asExpr().(WhenExpr).getBranch(_).getAResult() = node1.asExpr()
+  simpleAstFlowStep(node1.asExpr(), node2.asExpr())
   or
   exists(MethodAccess ma, ValuePreservingMethod m, int argNo |
     ma.getCallee().getSourceDeclaration() = m and m.returnsValue(argNo)
@@ -185,6 +201,8 @@ private predicate simpleLocalFlowStep0(Node node1, Node node2) {
   or
   FlowSummaryImpl::Private::Steps::summaryLocalStep(node1.(FlowSummaryNode).getSummaryNode(),
     node2.(FlowSummaryNode).getSummaryNode(), true)
+  or
+  captureValueStep(node1, node2)
 }
 
 /**
@@ -256,6 +274,19 @@ class MapValueContent extends Content, TMapValueContent {
   override string toString() { result = "<map.value>" }
 }
 
+/** A captured variable. */
+class CapturedVariableContent extends Content, TCapturedVariableContent {
+  CapturedVariable v;
+
+  CapturedVariableContent() { this = TCapturedVariableContent(v) }
+
+  CapturedVariable getVariable() { result = v }
+
+  override DataFlowType getType() { result = getErasedRepr(v.(Variable).getType()) }
+
+  override string toString() { result = v.toString() }
+}
+
 /** A reference through a synthetic instance field. */
 class SyntheticFieldContent extends Content, TSyntheticFieldContent {
   SyntheticField s;

java/ql/lib/semmle/code/java/dispatch/WrappedInvocation.qll

@@ -58,3 +58,37 @@ Method getRunnerTarget(MethodAccess ma) {
     result.overridesOrInstantiates*(runmethod)
   )
 }
+
+import semmle.code.java.dataflow.FlowSummary
+import semmle.code.java.dataflow.internal.FlowSummaryImplSpecific as ImplSpecific
+
+private predicate mayInvokeCallback(SrcMethod m, int n) {
+  m.getParameterType(n).(RefType).getSourceDeclaration() instanceof FunctionalInterface and
+  (not m.fromSource() or m.isNative() or m.getFile().getAbsolutePath().matches("%/test/stubs/%"))
+}
+
+private class SummarizedCallableWithCallback extends SummarizedCallable {
+  private int pos;
+
+  SummarizedCallableWithCallback() { mayInvokeCallback(this.asCallable(), pos) }
+
+  override predicate propagatesFlow(
+    SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
+  ) {
+    input = SummaryComponentStack::argument(pos) and
+    output = SummaryComponentStack::push(SummaryComponent::parameter(-1), input) and
+    preservesValue = true
+  }
+
+  override predicate hasProvenance(Provenance provenance) { provenance = "hq-generated" }
+}
+
+private class RequiredComponentStackForCallback extends RequiredSummaryComponentStack {
+  override predicate required(SummaryComponent head, SummaryComponentStack tail) {
+    exists(int pos |
+      mayInvokeCallback(_, pos) and
+      head = SummaryComponent::parameter(-1) and
+      tail = SummaryComponentStack::argument(pos)
+    )
+  }
+}

java/ql/lib/semmle/code/java/security/RandomDataSource.qll

@@ -103,7 +103,7 @@ class StdlibRandomSource extends RandomDataSource {
   }
 
   override Expr getOutput() {
-    if m.hasName("getBytes") then result = this.getArgument(0) else result = this
+    if m.hasName("nextBytes") then result = this.getArgument(0) else result = this
   }
 }
 

java/ql/src/Metrics/Summaries/TopJdkApis.qll

@@ -307,6 +307,7 @@ class TopJdkApi extends SummarizedCallableBase {
   predicate hasManualMadModel() { this.hasManualSummary() or this.hasManualNeutral() }
   /*
    * Note: the following top JDK APIs are not modeled with MaD:
+   * `java.lang.Runnable#run()`: specialised lambda flow
    * `java.lang.String#valueOf(Object)`: a complex case; an alias for `Object.toString`, except the dispatch is hidden
    * `java.lang.System#getProperty(String)`: needs to be modeled by regular CodeQL matching the get and set keys to reduce FPs
    * `java.lang.System#setProperty(String,String)`: needs to be modeled by regular CodeQL matching the get and set keys to reduce FPs

java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql

@@ -69,6 +69,7 @@ where
   // modeled in a MaD model, then it doesn't belong to any additional sink types, and we don't need to reexamine it.
   not CharacteristicsImpl::isSink(endpoint, _, _) and
   meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input, isVarargsArray) and
+  includeAutomodelCandidate(package, type, name, signature) and
   // The message is the concatenation of all sink types for which this endpoint is known neither to be a sink nor to be
   // a non-sink, and we surface only endpoints that have at least one such sink type.
   message =

java/ql/src/Telemetry/AutomodelCandidateFilter.yml

@@ -0,0 +1,5 @@
+extensions:
+  - addsTo:
+      pack: codeql/java-queries
+      extensible: automodelCandidateFilter
+    data: []

java/ql/src/Telemetry/AutomodelFrameworkModeExtractCandidates.ql

@@ -30,6 +30,7 @@ where
   // modeled in a MaD model, then it doesn't belong to any additional sink types, and we don't need to reexamine it.
   not CharacteristicsImpl::isSink(endpoint, _, _) and
   meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input, parameterName) and
+  includeAutomodelCandidate(package, type, name, signature) and
   // The message is the concatenation of all sink types for which this endpoint is known neither to be a sink nor to be
   // a non-sink, and we surface only endpoints that have at least one such sink type.
   message =

java/ql/src/Telemetry/AutomodelJavaUtil.qll

@@ -66,3 +66,24 @@ boolean considerSubtypes(Callable callable) {
   then result = false
   else result = true
 }
+
+/**
+ * Holds if the given package, type, name and signature is a candidate for automodeling.
+ *
+ * This predicate is extensible, so that different endpoints can be selected at runtime.
+ */
+extensible predicate automodelCandidateFilter(
+  string package, string type, string name, string signature
+);
+
+/**
+ * Holds if the given package, type, name and signature is a candidate for automodeling.
+ *
+ * This relies on an extensible predicate, and if that is not supplied then
+ * all endpoints are considered candidates.
+ */
+bindingset[package, type, name, signature]
+predicate includeAutomodelCandidate(string package, string type, string name, string signature) {
+  not automodelCandidateFilter(_, _, _, _) or
+  automodelCandidateFilter(package, type, name, signature)
+}

java/ql/src/qlpack.yml

@@ -12,4 +12,5 @@ dependencies:
   codeql/util: ${workspace}
 dataExtensions:
   - Telemetry/ExtractorInformation.yml
+  - Telemetry/AutomodelCandidateFilter.yml
 warnOnImplicitThis: true

java/ql/test/ext/TopJdkApis/TopJdkApisTest.expected

@@ -1,3 +1,4 @@
+| java.lang.Runnable#run() | no manual model |
 | java.lang.String#valueOf(Object) | no manual model |
 | java.lang.System#getProperty(String) | no manual model |
 | java.lang.System#setProperty(String,String) | no manual model |

java/ql/test/library-tests/dataflow/capture/B.java

@@ -0,0 +1,251 @@
+import java.util.*;
+import java.util.function.*;
+
+public class B {
+  static String source(String label) { return null; }
+
+  static void sink(String s) { }
+
+  static void test1() {
+    List<String> l1 = new ArrayList<>();
+    l1.add(source("L"));
+    List<String> l2 = new ArrayList<>();
+    l1.forEach(e -> l2.add(e));
+    sink(l2.get(0)); // $ hasValueFlow=L
+  }
+
+  String bf1;
+  String bf2;
+
+  void test2() {
+    B other = new B();
+    Consumer<String> f = x -> { this.bf1 = x; bf2 = x; other.bf1 = x; };
+
+    // no flow
+    sink(bf1);
+    sink(this.bf2);
+    sink(other.bf1);
+    sink(other.bf2);
+
+    f.accept(source("T"));
+
+    sink(bf1); // $ MISSING: hasValueFlow=T
+    sink(this.bf2); // $ MISSING: hasValueFlow=T
+    sink(other.bf1); // $ hasValueFlow=T
+    sink(other.bf2);
+  }
+
+  static void convert(Map<String, String> inp, Map<String, String> out) {
+    inp.forEach((key, value) -> { out.put(key, value); });
+  }
+
+  void test3() {
+    HashMap<String,String> m1 = new HashMap<>();
+    HashMap<String,String> m2 = new HashMap<>();
+    m1.put(source("Key"), source("Value"));
+    convert(m1, m2);
+    m2.forEach((k, v) -> {
+      sink(k); // $ hasValueFlow=Key
+      sink(v); // $ hasValueFlow=Value
+    });
+  }
+
+  String elem;
+
+  void testParamIn1() {
+    elem = source("pin.This.elem");
+    testParamIn2(source("pin.Arg"));
+  }
+
+  void testParamIn2(String param) {
+    Runnable r = () -> {
+      sink(elem); // $ MISSING: hasValueFlow=pin.This.elem
+      sink(this.elem); // $ MISSING: hasValueFlow=pin.This.elem
+      sink(param); // $ hasValueFlow=pin.Arg
+    };
+    r.run();
+  }
+
+  void testParamOut1() {
+    B other = new B();
+    testParamOut2(other);
+    sink(elem); // $ MISSING: hasValueFlow=pout.This.elem
+    sink(this.elem); // $ MISSING: hasValueFlow=pout.This.elem
+    sink(other.elem); // $ hasValueFlow=pout.param
+  }
+
+  void testParamOut2(B param) {
+    Runnable r = () -> {
+      this.elem = source("pout.This.elem");
+      param.elem = source("pout.param");
+    };
+    r.run();
+  }
+
+  void testCrossLambda() {
+    B b = new B();
+    Runnable sink1 = () -> { sink(b.elem); };
+    Runnable sink2 = () -> { sink(b.elem); }; // $ hasValueFlow=src
+    Runnable src = () -> { b.elem = source("src"); };
+    doRun(sink1);
+    doRun(src);
+    doRun(sink2);
+  }
+
+  void doRun(Runnable r) {
+    r.run();
+  }
+
+  void testNested() {
+    List<String> l1 = new ArrayList<>();
+    List<List<String>> l2 = new ArrayList<>();
+    l1.add(source("nest.out"));
+    l2.add(l1);
+    String s = source("nest.in");
+    List<String> out1 = new ArrayList<>();
+    List<String> out2 = new ArrayList<>();
+    l2.forEach(l -> l.forEach(x -> {
+      sink(s); // $ hasValueFlow=nest.in
+      out1.add(x);
+      out2.add(s);
+    }));
+    sink(out1.get(0)); // $ hasValueFlow=nest.out
+    sink(out2.get(0)); // $ hasValueFlow=nest.in
+  }
+
+  static interface TwoRuns {
+    void run1();
+    void run2();
+  }
+
+  void testAnonymousClass() {
+    List<String> l1 = new ArrayList<>();
+    List<String> l2 = new ArrayList<>();
+    TwoRuns r = new TwoRuns() {
+      @Override
+      public void run1() {
+        l1.add(source("run1"));
+      }
+      @Override
+      public void run2() {
+        l2.add(l1.get(0));
+      }
+    };
+    r.run2();
+    sink(l2.get(0));
+    r.run1();
+    r.run2();
+    sink(l2.get(0)); // $ hasValueFlow=run1
+  }
+
+  void testLocalClass1() {
+    String s = source("local1");
+    class MyLocal {
+      String f;
+      MyLocal() { this.f = s; }
+      String getF() { return this.f; }
+    }
+    MyLocal m = new MyLocal();
+    sink(m.getF()); // $ hasValueFlow=local1
+  }
+
+  void testLocalClass2() {
+    String s1 = source("s1");
+    String s2 = source("s2");
+    List<String> l = new ArrayList<>();
+    class MyLocal {
+      String f;
+      MyLocal() {
+        this.f = s1;
+        sink(s2); // $ hasValueFlow=s2
+      }
+      void test() {
+        sink(f); // $ hasValueFlow=s1
+        sink(s2); // $ hasValueFlow=s2
+      }
+      void add(String s) {
+        l.add(s);
+      }
+      String get() {
+        return l.get(0);
+      }
+    }
+    MyLocal m1 = new MyLocal();
+    MyLocal m2 = new MyLocal();
+    m1.test();
+    sink(m1.get());
+    m1.add(source("m1.add"));
+    sink(m2.get()); // $ hasValueFlow=m1.add
+  }
+
+  void testComplex() {
+    String s = source("complex");
+    class LocalComplex {
+      Supplier<StringBox> getBoxSupplier() {
+        return new Supplier<StringBox>() {
+          StringBox b = new StringBox();
+          @Override
+          public StringBox get() { return b; }
+        };
+      }
+      class StringBox {
+        String get() {
+          // capture through regular nested class inside local nested class
+          return s;
+        }
+      }
+    }
+    LocalComplex lc = new LocalComplex();
+    sink(lc.getBoxSupplier().get().get()); // $ MISSING: hasValueFlow=complex
+  }
+
+  void testCapturedLambda() {
+    String s = source("double.capture.in");
+    List<String> out = new ArrayList<>();
+    Runnable r1 = () -> {
+      sink(s); // $ hasValueFlow=double.capture.in
+      out.add(source("double.capture.out"));
+    };
+    Runnable r2 = () -> {
+      r1.run();
+    };
+    r2.run();
+    sink(out.get(0)); // $ MISSING: hasValueFlow=double.capture.out
+  }
+
+  void testEnhancedForStmtCapture() {
+    List<String> l = new ArrayList<>();
+    l.add(source("list"));
+    String[] a = new String[] { source("array") };
+    for (String x : l) {
+      Runnable r = () -> sink(x); // $ MISSING: hasValueFlow=list
+      r.run();
+    }
+    for (String x : a) {
+      Runnable r = () -> sink(x); // $ MISSING: hasValueFlow=array
+      r.run();
+    }
+  }
+
+  void testDoubleCall() {
+    String s = source("src");
+    List<String> l = new ArrayList<>();
+    List<String> l2 = new ArrayList<>();
+    class MyLocal2 {
+      MyLocal2() {
+        sink(l.get(0)); // no flow
+        sink(l2.get(0)); // no flow
+        l.add(s);
+      }
+      void run() {
+        l2.add(l.get(0));
+      }
+    }
+    // The ClassInstanceExpr has two calls in the same cfg node:
+    // First the constructor call for which it is the postupdate,
+    // and then as instance argument to the run call.
+    new MyLocal2().run();
+    sink(l.get(0)); // $ hasValueFlow=src
+    sink(l2.get(0)); // $ hasValueFlow=src
+  }
+}

java/ql/test/library-tests/dataflow/capture/inlinetest.expected

@@ -0,0 +1,2 @@
+failures
+testFailures

java/ql/test/library-tests/dataflow/capture/inlinetest.ql

@@ -0,0 +1,2 @@
+import TestUtilities.InlineFlowTest
+import DefaultFlowTest

java/ql/test/library-tests/dataflow/capture/test.expected

@@ -1,26 +1,93 @@
-| A.java:14:14:14:16 | "A" | A.java:14:14:14:16 | "A" |
-| A.java:14:14:14:16 | "A" | A.java:15:16:15:22 | get(...) |
-| A.java:14:14:14:16 | "A" | A.java:18:8:18:15 | p |
-| A.java:14:14:14:16 | "A" | A.java:32:26:32:26 | p |
-| A.java:21:11:21:13 | "B" | A.java:15:16:15:22 | get(...) |
-| A.java:21:11:21:13 | "B" | A.java:21:7:21:13 | ...=... |
-| A.java:21:11:21:13 | "B" | A.java:21:11:21:13 | "B" |
-| A.java:21:11:21:13 | "B" | A.java:33:26:33:26 | s |
-| A.java:23:11:23:13 | "C" | A.java:15:16:15:22 | get(...) |
-| A.java:23:11:23:13 | "C" | A.java:23:7:23:13 | ...=... |
-| A.java:23:11:23:13 | "C" | A.java:23:11:23:13 | "C" |
-| A.java:23:11:23:13 | "C" | A.java:33:26:33:26 | s |
-| A.java:25:22:25:24 | "D" | A.java:4:9:4:16 | e |
-| A.java:25:22:25:24 | "D" | A.java:4:21:4:28 | ...=... |
-| A.java:25:22:25:24 | "D" | A.java:4:28:4:28 | e |
-| A.java:25:22:25:24 | "D" | A.java:6:31:6:34 | elem |
-| A.java:25:22:25:24 | "D" | A.java:15:16:15:22 | get(...) |
-| A.java:25:22:25:24 | "D" | A.java:25:22:25:24 | "D" |
-| A.java:25:22:25:24 | "D" | A.java:34:26:34:37 | getElem(...) |
-| A.java:27:16:27:18 | "E" | A.java:5:18:5:25 | e |
-| A.java:27:16:27:18 | "E" | A.java:5:30:5:37 | ...=... |
-| A.java:27:16:27:18 | "E" | A.java:5:37:5:37 | e |
-| A.java:27:16:27:18 | "E" | A.java:6:31:6:34 | elem |
-| A.java:27:16:27:18 | "E" | A.java:15:16:15:22 | get(...) |
-| A.java:27:16:27:18 | "E" | A.java:27:16:27:18 | "E" |
-| A.java:27:16:27:18 | "E" | A.java:35:26:35:37 | getElem(...) |
+| A.java:14:14:14:16 | "A" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:15:16:15:16 | a : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:15:16:15:22 | get(...) : String |
+| A.java:14:14:14:16 | "A" : String | A.java:18:8:18:15 | p : String |
+| A.java:14:14:14:16 | "A" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:28:11:38:5 | p : String |
+| A.java:14:14:14:16 | "A" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:31:17:31:17 | this : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:32:26:32:26 | p : String |
+| A.java:14:14:14:16 | "A" : String | A.java:32:26:32:26 | this : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:33:26:33:26 | this : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:34:26:34:27 | this : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:35:26:35:27 | this : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:39:12:39:12 | a : new A(...) { ... } [p] |
+| A.java:14:14:14:16 | "A" : String | A.java:39:12:39:12 | p : String |
+| A.java:21:11:21:13 | "B" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:15:16:15:16 | a : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:15:16:15:22 | get(...) : String |
+| A.java:21:11:21:13 | "B" : String | A.java:21:7:21:13 | ...=... : String |
+| A.java:21:11:21:13 | "B" : String | A.java:25:5:25:26 | phi(String s) : String |
+| A.java:21:11:21:13 | "B" : String | A.java:28:11:38:5 | String s : String |
+| A.java:21:11:21:13 | "B" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:31:17:31:17 | this : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:32:26:32:26 | this : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:33:26:33:26 | s : String |
+| A.java:21:11:21:13 | "B" : String | A.java:33:26:33:26 | this : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:34:26:34:27 | this : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:35:26:35:27 | this : new A(...) { ... } [String s] |
+| A.java:21:11:21:13 | "B" : String | A.java:39:12:39:12 | String s : String |
+| A.java:21:11:21:13 | "B" : String | A.java:39:12:39:12 | a : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:15:16:15:16 | a : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:15:16:15:22 | get(...) : String |
+| A.java:23:11:23:13 | "C" : String | A.java:23:7:23:13 | ...=... : String |
+| A.java:23:11:23:13 | "C" : String | A.java:25:5:25:26 | phi(String s) : String |
+| A.java:23:11:23:13 | "C" : String | A.java:28:11:38:5 | String s : String |
+| A.java:23:11:23:13 | "C" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:31:17:31:17 | this : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:32:26:32:26 | this : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:33:26:33:26 | s : String |
+| A.java:23:11:23:13 | "C" : String | A.java:33:26:33:26 | this : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:34:26:34:27 | this : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:35:26:35:27 | this : new A(...) { ... } [String s] |
+| A.java:23:11:23:13 | "C" : String | A.java:39:12:39:12 | String s : String |
+| A.java:23:11:23:13 | "C" : String | A.java:39:12:39:12 | a : new A(...) { ... } [String s] |
+| A.java:25:22:25:24 | "D" : String | A.java:4:9:4:16 | e : String |
+| A.java:25:22:25:24 | "D" : String | A.java:4:21:4:24 | this <.field> [post update] : Box [elem] |
+| A.java:25:22:25:24 | "D" : String | A.java:4:21:4:28 | ...=... : String |
+| A.java:25:22:25:24 | "D" : String | A.java:4:28:4:28 | e : String |
+| A.java:25:22:25:24 | "D" : String | A.java:6:12:6:18 | parameter this : Box [elem] |
+| A.java:25:22:25:24 | "D" : String | A.java:6:31:6:34 | elem : String |
+| A.java:25:22:25:24 | "D" : String | A.java:6:31:6:34 | this <.field> : Box [elem] |
+| A.java:25:22:25:24 | "D" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:15:16:15:16 | a : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:15:16:15:22 | get(...) : String |
+| A.java:25:22:25:24 | "D" : String | A.java:25:14:25:25 | new Box(...) : Box [elem] |
+| A.java:25:22:25:24 | "D" : String | A.java:28:11:38:5 | Box b1 : Box [elem] |
+| A.java:25:22:25:24 | "D" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:31:17:31:17 | this : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:32:26:32:26 | this : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:33:26:33:26 | this : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:34:26:34:27 | b1 : Box [elem] |
+| A.java:25:22:25:24 | "D" : String | A.java:34:26:34:27 | this : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:34:26:34:37 | getElem(...) : String |
+| A.java:25:22:25:24 | "D" : String | A.java:35:26:35:27 | this : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:25:22:25:24 | "D" : String | A.java:39:12:39:12 | Box b1 : Box [elem] |
+| A.java:25:22:25:24 | "D" : String | A.java:39:12:39:12 | a : new A(...) { ... } [Box b1, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:5:18:5:25 | e : String |
+| A.java:27:16:27:18 | "E" : String | A.java:5:30:5:33 | this <.field> [post update] : Box [elem] |
+| A.java:27:16:27:18 | "E" : String | A.java:5:30:5:37 | ...=... : String |
+| A.java:27:16:27:18 | "E" : String | A.java:5:37:5:37 | e : String |
+| A.java:27:16:27:18 | "E" : String | A.java:6:12:6:18 | parameter this : Box [elem] |
+| A.java:27:16:27:18 | "E" : String | A.java:6:31:6:34 | elem : String |
+| A.java:27:16:27:18 | "E" : String | A.java:6:31:6:34 | this <.field> : Box [elem] |
+| A.java:27:16:27:18 | "E" : String | A.java:14:11:14:20 | f2(...) : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:15:16:15:16 | a : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:15:16:15:22 | get(...) : String |
+| A.java:27:16:27:18 | "E" : String | A.java:27:5:27:6 | b2 [post update] : Box [elem] |
+| A.java:27:16:27:18 | "E" : String | A.java:28:11:38:5 | Box b2 : Box [elem] |
+| A.java:27:16:27:18 | "E" : String | A.java:28:11:38:5 | new (...) : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:30:14:30:16 | parameter this : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:31:17:31:17 | this : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:32:26:32:26 | this : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:33:26:33:26 | this : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:34:26:34:27 | this : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:35:26:35:27 | b2 : Box [elem] |
+| A.java:27:16:27:18 | "E" : String | A.java:35:26:35:27 | this : new A(...) { ... } [Box b2, ... (2)] |
+| A.java:27:16:27:18 | "E" : String | A.java:35:26:35:37 | getElem(...) : String |
+| A.java:27:16:27:18 | "E" : String | A.java:39:12:39:12 | Box b2 : Box [elem] |
+| A.java:27:16:27:18 | "E" : String | A.java:39:12:39:12 | a : new A(...) { ... } [Box b2, ... (2)] |

java/ql/test/library-tests/dataflow/capture/test.ql

@@ -1,16 +1,23 @@
 import java
 import semmle.code.java.dataflow.DataFlow
 
-StringLiteral src() { result.getCompilationUnit().fromSource() }
+StringLiteral src() {
+  result.getCompilationUnit().fromSource() and
+  result.getFile().toString() = "A"
+}
 
 module Config implements DataFlow::ConfigSig {
   predicate isSource(DataFlow::Node n) { n.asExpr() = src() }
 
-  predicate isSink(DataFlow::Node n) { any() }
+  predicate isSink(DataFlow::Node n) { none() }
 }
 
 module Flow = DataFlow::Global<Config>;
 
-from DataFlow::Node src, DataFlow::Node sink
-where Flow::flow(src, sink)
+int explorationLimit() { result = 100 }
+
+module PartialFlow = Flow::FlowExploration<explorationLimit/0>;
+
+from PartialFlow::PartialPathNode src, PartialFlow::PartialPathNode sink
+where PartialFlow::partialFlow(src, sink, _)
 select src, sink

java/ql/test/library-tests/dataflow/stream-read/A.java

@@ -99,7 +99,7 @@ public class A {
     }
 
     public static void testWrapCall() {
-        sink(wrapStream(null)); // $ SPURIOUS: hasTaintFlow
+        sink(wrapStream(null)); // no flow
         sink(wrapStream(source())); // $ hasTaintFlow
     }
 

java/ql/test/library-tests/dataflow/threat-models/Empty.java

@@ -0,0 +1 @@
+class Empty { }

java/ql/test/library-tests/dataflow/threat-models/threat-models1.expected

@@ -0,0 +1,5 @@
+| default |
+| remote |
+| request |
+| response |
+| uri-path |

java/ql/test/library-tests/dataflow/threat-models/threat-models1.ql

@@ -0,0 +1,5 @@
+import semmle.code.java.dataflow.ExternalFlowConfiguration as ExternalFlowConfiguration
+
+query predicate supportedThreatModels(string kind) {
+  ExternalFlowConfiguration::sourceModelKindConfig(kind)
+}

java/ql/test/library-tests/dataflow/threat-models/threat-models2.expected

@@ -0,0 +1,10 @@
+| cli |
+| database |
+| default |
+| environment |
+| file |
+| local |
+| remote |
+| request |
+| response |
+| uri-path |

java/ql/test/library-tests/dataflow/threat-models/threat-models2.ext.yml

@@ -0,0 +1,7 @@
+extensions:
+
+  - addsTo:
+      pack: codeql/java-all
+      extensible: supportedThreatModels
+    data:
+      - ["local"] # Add the "local" group threat model.

java/ql/test/library-tests/dataflow/threat-models/threat-models2.ql

@@ -0,0 +1,5 @@
+import semmle.code.java.dataflow.ExternalFlowConfiguration as ExternalFlowConfiguration
+
+query predicate supportedThreatModels(string kind) {
+  ExternalFlowConfiguration::sourceModelKindConfig(kind)
+}

java/ql/test/library-tests/frameworks/ratpack/resources/IntegrationTest.java

@@ -107,13 +107,13 @@ class IntegrationTest {
                 filterAndMerge_2(pojoForm, mergedParams, name -> false);
                 return mergedParams;
             }).then(pojoMap -> {
-                sink(pojoMap.keySet().iterator().next()); //TODO:$hasTaintFlow
-                sink(pojoMap.get("value")); //TODO:$hasTaintFlow
+                sink(pojoMap.keySet().iterator().next()); //$hasTaintFlow
+                sink(pojoMap.get("value")); //$hasTaintFlow
                 pojoMap.forEach((key, value) -> {
-                    sink(key); //TODO:$hasTaintFlow
-                    sink(value); //TODO:$hasTaintFlow
+                    sink(key); //$hasTaintFlow
+                    sink(value); //$hasTaintFlow
                     List<Object> values = (List<Object>) value;
-                    sink(values.get(0)); //TODO:$hasTaintFlow
+                    sink(values.get(0)); //$hasTaintFlow
                 });
             });
     }

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll

@@ -1,3 +1,3 @@
 private import DataFlowImplSpecific
-private import codeql.dataflow.DataFlowImpl
+private import codeql.dataflow.internal.DataFlowImpl
 import MakeImpl<PythonDataFlow>

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll

@@ -1,3 +1,3 @@
 private import DataFlowImplSpecific
-private import codeql.dataflow.DataFlowImplCommon
+private import codeql.dataflow.internal.DataFlowImplCommon
 import MakeImplCommon<PythonDataFlow>

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplSpecific.qll

@@ -2,7 +2,7 @@
  * Provides Python-specific definitions for use in the data flow library.
  */
 
-private import codeql.dataflow.DataFlowParameter
+private import codeql.dataflow.DataFlow
 // we need to export `Unit` for the DataFlowImpl* files
 private import python as Python
 
@@ -15,7 +15,7 @@ module Public {
   import DataFlowUtil
 }
 
-module PythonDataFlow implements DataFlowParameter {
+module PythonDataFlow implements InputSig {
   import Private
   import Public
 

ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll

@@ -1,3 +1,3 @@
 private import DataFlowImplSpecific
-private import codeql.dataflow.DataFlowImpl
+private import codeql.dataflow.internal.DataFlowImpl
 import MakeImpl<RubyDataFlow>

ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll

@@ -1,3 +1,3 @@
 private import DataFlowImplSpecific
-private import codeql.dataflow.DataFlowImplCommon
+private import codeql.dataflow.internal.DataFlowImplCommon
 import MakeImplCommon<RubyDataFlow>

ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplSpecific.qll

@@ -2,7 +2,7 @@
  * Provides Ruby-specific definitions for use in the data flow library.
  */
 
-private import codeql.dataflow.DataFlowParameter
+private import codeql.dataflow.DataFlow
 
 module Private {
   import DataFlowPrivate
@@ -13,7 +13,7 @@ module Public {
   import DataFlowPublic
 }
 
-module RubyDataFlow implements DataFlowParameter {
+module RubyDataFlow implements InputSig {
   import Private
   import Public
 

shared/dataflow/change-notes/2023-06-16-initial-version.md

@@ -0,0 +1,4 @@
+---
+category: majorAnalysis
+---
+* Initial release. Adds a library to implement flow through captured variables that properly adheres to inter-procedural control flow.

shared/dataflow/codeql/dataflow/DataFlow.qll

@@ -1,15 +1,234 @@
 /**
  * Provides an implementation of global (interprocedural) data flow. This file
- * re-exports the local (intraprocedural) data flow analysis from
- * `DataFlowImplSpecific::Public` and adds a global analysis, mainly exposed
- * through the `Global` and `GlobalWithState` modules.
+ * adds a global analysis, mainly exposed through the `Global` and `GlobalWithState`
+ * modules.
  */
 
-import DataFlowParameter
+/** Provides language-specific data flow parameters. */
+signature module InputSig {
+  class Node {
+    /** Gets a textual representation of this element. */
+    string toString();
 
-module Configs<DataFlowParameter Lang> {
+    /**
+     * Holds if this element is at the specified location.
+     * The location spans column `startcolumn` of line `startline` to
+     * column `endcolumn` of line `endline` in file `filepath`.
+     * For more information, see
+     * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
+     */
+    predicate hasLocationInfo(
+      string filepath, int startline, int startcolumn, int endline, int endcolumn
+    );
+  }
+
+  class ParameterNode extends Node;
+
+  class ArgumentNode extends Node;
+
+  class ReturnNode extends Node {
+    ReturnKind getKind();
+  }
+
+  class OutNode extends Node;
+
+  class PostUpdateNode extends Node {
+    Node getPreUpdateNode();
+  }
+
+  class CastNode extends Node;
+
+  predicate isParameterNode(ParameterNode p, DataFlowCallable c, ParameterPosition pos);
+
+  predicate isArgumentNode(ArgumentNode n, DataFlowCall call, ArgumentPosition pos);
+
+  DataFlowCallable nodeGetEnclosingCallable(Node node);
+
+  DataFlowType getNodeType(Node node);
+
+  predicate nodeIsHidden(Node node);
+
+  class DataFlowExpr;
+
+  /** Gets the node corresponding to `e`. */
+  Node exprNode(DataFlowExpr e);
+
+  class DataFlowCall {
+    /** Gets a textual representation of this element. */
+    string toString();
+
+    DataFlowCallable getEnclosingCallable();
+  }
+
+  class DataFlowCallable {
+    /** Gets a textual representation of this element. */
+    string toString();
+  }
+
+  class ReturnKind {
+    /** Gets a textual representation of this element. */
+    string toString();
+  }
+
+  /** Gets a viable implementation of the target of the given `Call`. */
+  DataFlowCallable viableCallable(DataFlowCall c);
+
+  /**
+   * Holds if the set of viable implementations that can be called by `call`
+   * might be improved by knowing the call context.
+   */
+  predicate mayBenefitFromCallContext(DataFlowCall call, DataFlowCallable c);
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference.
+   */
+  DataFlowCallable viableImplInCallContext(DataFlowCall call, DataFlowCall ctx);
+
+  /**
+   * Gets a node that can read the value returned from `call` with return kind
+   * `kind`.
+   */
+  OutNode getAnOutNode(DataFlowCall call, ReturnKind kind);
+
+  class DataFlowType {
+    /** Gets a textual representation of this element. */
+    string toString();
+  }
+
+  string ppReprType(DataFlowType t);
+
+  bindingset[t1, t2]
+  predicate compatibleTypes(DataFlowType t1, DataFlowType t2);
+
+  predicate typeStrongerThan(DataFlowType t1, DataFlowType t2);
+
+  class Content {
+    /** Gets a textual representation of this element. */
+    string toString();
+  }
+
+  predicate forceHighPrecision(Content c);
+
+  /**
+   * An entity that represents a set of `Content`s.
+   *
+   * The set may be interpreted differently depending on whether it is
+   * stored into (`getAStoreContent`) or read from (`getAReadContent`).
+   */
+  class ContentSet {
+    /** Gets a content that may be stored into when storing into this set. */
+    Content getAStoreContent();
+
+    /** Gets a content that may be read from when reading from this set. */
+    Content getAReadContent();
+  }
+
+  class ContentApprox {
+    /** Gets a textual representation of this element. */
+    string toString();
+  }
+
+  ContentApprox getContentApprox(Content c);
+
+  class ParameterPosition {
+    /** Gets a textual representation of this element. */
+    bindingset[this]
+    string toString();
+  }
+
+  class ArgumentPosition {
+    /** Gets a textual representation of this element. */
+    bindingset[this]
+    string toString();
+  }
+
+  predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos);
+
+  predicate simpleLocalFlowStep(Node node1, Node node2);
+
+  /**
+   * Holds if data can flow from `node1` to `node2` through a non-local step
+   * that does not follow a call edge. For example, a step through a global
+   * variable.
+   */
+  predicate jumpStep(Node node1, Node node2);
+
+  /**
+   * Holds if data can flow from `node1` to `node2` via a read of `c`.  Thus,
+   * `node1` references an object with a content `c.getAReadContent()` whose
+   * value ends up in `node2`.
+   */
+  predicate readStep(Node node1, ContentSet c, Node node2);
+
+  /**
+   * Holds if data can flow from `node1` to `node2` via a store into `c`.  Thus,
+   * `node2` references an object with a content `c.getAStoreContent()` that
+   * contains the value of `node1`.
+   */
+  predicate storeStep(Node node1, ContentSet c, Node node2);
+
+  /**
+   * Holds if values stored inside content `c` are cleared at node `n`. For example,
+   * any value stored inside `f` is cleared at the pre-update node associated with `x`
+   * in `x.f = newValue`.
+   */
+  predicate clearsContent(Node n, ContentSet c);
+
+  /**
+   * Holds if the value that is being tracked is expected to be stored inside content `c`
+   * at node `n`.
+   */
+  predicate expectsContent(Node n, ContentSet c);
+
+  /**
+   * Holds if the node `n` is unreachable when the call context is `call`.
+   */
+  predicate isUnreachableInCall(Node n, DataFlowCall call);
+
+  default int accessPathLimit() { result = 5 }
+
+  /**
+   * Holds if flow is allowed to pass from parameter `p` and back to itself as a
+   * side-effect, resulting in a summary from `p` to itself.
+   *
+   * One example would be to allow flow like `p.foo = p.bar;`, which is disallowed
+   * by default as a heuristic.
+   */
+  predicate allowParameterReturnInSelf(ParameterNode p);
+
+  class LambdaCallKind;
+
+  /** Holds if `creation` is an expression that creates a lambda of kind `kind` for `c`. */
+  predicate lambdaCreation(Node creation, LambdaCallKind kind, DataFlowCallable c);
+
+  /** Holds if `call` is a lambda call of kind `kind` where `receiver` is the lambda expression. */
+  predicate lambdaCall(DataFlowCall call, LambdaCallKind kind, Node receiver);
+
+  /** Extra data-flow steps needed for lambda flow analysis. */
+  predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preservesValue);
+
+  /**
+   * Holds if `n` should never be skipped over in the `PathGraph` and in path
+   * explanations.
+   */
+  default predicate neverSkipInPathGraph(Node n) { none() }
+
+  /**
+   * Gets an additional term that is added to the `join` and `branch` computations to reflect
+   * an additional forward or backwards branching factor that is not taken into account
+   * when calculating the (virtual) dispatch cost.
+   *
+   * Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter.
+   */
+  int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p);
+
+  predicate golangSpecificParamArgFilter(DataFlowCall call, ParameterNode p, ArgumentNode arg);
+}
+
+module Configs<InputSig Lang> {
   private import Lang
-  private import DataFlowImplCommon::MakeImplCommon<Lang>
+  private import internal.DataFlowImplCommon::MakeImplCommon<Lang>
   import DataFlowImplCommonPublic
 
   /** An input configuration for data flow. */
@@ -211,9 +430,9 @@ module Configs<DataFlowParameter Lang> {
   }
 }
 
-module DataFlowMake<DataFlowParameter Lang> {
+module DataFlowMake<InputSig Lang> {
   private import Lang
-  private import DataFlowImpl::MakeImpl<Lang>
+  private import internal.DataFlowImpl::MakeImpl<Lang>
   import Configs<Lang>
 
   /**

shared/dataflow/codeql/dataflow/DataFlowParameter.qll

@@ -1,220 +0,0 @@
-signature module DataFlowParameter {
-  class Node {
-    /** Gets a textual representation of this element. */
-    string toString();
-
-    /**
-     * Holds if this element is at the specified location.
-     * The location spans column `startcolumn` of line `startline` to
-     * column `endcolumn` of line `endline` in file `filepath`.
-     * For more information, see
-     * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
-     */
-    predicate hasLocationInfo(
-      string filepath, int startline, int startcolumn, int endline, int endcolumn
-    );
-  }
-
-  class ParameterNode extends Node;
-
-  class ArgumentNode extends Node;
-
-  class ReturnNode extends Node {
-    ReturnKind getKind();
-  }
-
-  class OutNode extends Node;
-
-  class PostUpdateNode extends Node {
-    Node getPreUpdateNode();
-  }
-
-  class CastNode extends Node;
-
-  predicate isParameterNode(ParameterNode p, DataFlowCallable c, ParameterPosition pos);
-
-  predicate isArgumentNode(ArgumentNode n, DataFlowCall call, ArgumentPosition pos);
-
-  DataFlowCallable nodeGetEnclosingCallable(Node node);
-
-  DataFlowType getNodeType(Node node);
-
-  predicate nodeIsHidden(Node node);
-
-  class DataFlowExpr;
-
-  /** Gets the node corresponding to `e`. */
-  Node exprNode(DataFlowExpr e);
-
-  class DataFlowCall {
-    /** Gets a textual representation of this element. */
-    string toString();
-
-    DataFlowCallable getEnclosingCallable();
-  }
-
-  class DataFlowCallable {
-    /** Gets a textual representation of this element. */
-    string toString();
-  }
-
-  class ReturnKind {
-    /** Gets a textual representation of this element. */
-    string toString();
-  }
-
-  /** Gets a viable implementation of the target of the given `Call`. */
-  DataFlowCallable viableCallable(DataFlowCall c);
-
-  /**
-   * Holds if the set of viable implementations that can be called by `call`
-   * might be improved by knowing the call context.
-   */
-  predicate mayBenefitFromCallContext(DataFlowCall call, DataFlowCallable c);
-
-  /**
-   * Gets a viable dispatch target of `call` in the context `ctx`. This is
-   * restricted to those `call`s for which a context might make a difference.
-   */
-  DataFlowCallable viableImplInCallContext(DataFlowCall call, DataFlowCall ctx);
-
-  /**
-   * Gets a node that can read the value returned from `call` with return kind
-   * `kind`.
-   */
-  OutNode getAnOutNode(DataFlowCall call, ReturnKind kind);
-
-  class DataFlowType {
-    /** Gets a textual representation of this element. */
-    string toString();
-  }
-
-  string ppReprType(DataFlowType t);
-
-  bindingset[t1, t2]
-  predicate compatibleTypes(DataFlowType t1, DataFlowType t2);
-
-  predicate typeStrongerThan(DataFlowType t1, DataFlowType t2);
-
-  class Content {
-    /** Gets a textual representation of this element. */
-    string toString();
-  }
-
-  predicate forceHighPrecision(Content c);
-
-  /**
-   * An entity that represents a set of `Content`s.
-   *
-   * The set may be interpreted differently depending on whether it is
-   * stored into (`getAStoreContent`) or read from (`getAReadContent`).
-   */
-  class ContentSet {
-    /** Gets a content that may be stored into when storing into this set. */
-    Content getAStoreContent();
-
-    /** Gets a content that may be read from when reading from this set. */
-    Content getAReadContent();
-  }
-
-  class ContentApprox {
-    /** Gets a textual representation of this element. */
-    string toString();
-  }
-
-  ContentApprox getContentApprox(Content c);
-
-  class ParameterPosition {
-    /** Gets a textual representation of this element. */
-    bindingset[this]
-    string toString();
-  }
-
-  class ArgumentPosition {
-    /** Gets a textual representation of this element. */
-    bindingset[this]
-    string toString();
-  }
-
-  predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos);
-
-  predicate simpleLocalFlowStep(Node node1, Node node2);
-
-  /**
-   * Holds if data can flow from `node1` to `node2` through a non-local step
-   * that does not follow a call edge. For example, a step through a global
-   * variable.
-   */
-  predicate jumpStep(Node node1, Node node2);
-
-  /**
-   * Holds if data can flow from `node1` to `node2` via a read of `c`.  Thus,
-   * `node1` references an object with a content `c.getAReadContent()` whose
-   * value ends up in `node2`.
-   */
-  predicate readStep(Node node1, ContentSet c, Node node2);
-
-  /**
-   * Holds if data can flow from `node1` to `node2` via a store into `c`.  Thus,
-   * `node2` references an object with a content `c.getAStoreContent()` that
-   * contains the value of `node1`.
-   */
-  predicate storeStep(Node node1, ContentSet c, Node node2);
-
-  /**
-   * Holds if values stored inside content `c` are cleared at node `n`. For example,
-   * any value stored inside `f` is cleared at the pre-update node associated with `x`
-   * in `x.f = newValue`.
-   */
-  predicate clearsContent(Node n, ContentSet c);
-
-  /**
-   * Holds if the value that is being tracked is expected to be stored inside content `c`
-   * at node `n`.
-   */
-  predicate expectsContent(Node n, ContentSet c);
-
-  /**
-   * Holds if the node `n` is unreachable when the call context is `call`.
-   */
-  predicate isUnreachableInCall(Node n, DataFlowCall call);
-
-  default int accessPathLimit() { result = 5 }
-
-  /**
-   * Holds if flow is allowed to pass from parameter `p` and back to itself as a
-   * side-effect, resulting in a summary from `p` to itself.
-   *
-   * One example would be to allow flow like `p.foo = p.bar;`, which is disallowed
-   * by default as a heuristic.
-   */
-  predicate allowParameterReturnInSelf(ParameterNode p);
-
-  class LambdaCallKind;
-
-  /** Holds if `creation` is an expression that creates a lambda of kind `kind` for `c`. */
-  predicate lambdaCreation(Node creation, LambdaCallKind kind, DataFlowCallable c);
-
-  /** Holds if `call` is a lambda call of kind `kind` where `receiver` is the lambda expression. */
-  predicate lambdaCall(DataFlowCall call, LambdaCallKind kind, Node receiver);
-
-  /** Extra data-flow steps needed for lambda flow analysis. */
-  predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preservesValue);
-
-  /**
-   * Holds if `n` should never be skipped over in the `PathGraph` and in path
-   * explanations.
-   */
-  default predicate neverSkipInPathGraph(Node n) { none() }
-
-  /**
-   * Gets an additional term that is added to the `join` and `branch` computations to reflect
-   * an additional forward or backwards branching factor that is not taken into account
-   * when calculating the (virtual) dispatch cost.
-   *
-   * Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter.
-   */
-  int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p);
-
-  predicate golangSpecificParamArgFilter(DataFlowCall call, ParameterNode p, ArgumentNode arg);
-}

shared/dataflow/codeql/dataflow/VariableCapture.qll

@@ -0,0 +1,929 @@
+/**
+ * Provides a module for synthesizing data-flow nodes and related step relations
+ * for supporting flow through captured variables.
+ */
+
+private import codeql.util.Boolean
+private import codeql.util.Unit
+private import codeql.ssa.Ssa as Ssa
+
+signature module InputSig {
+  class Location {
+    predicate hasLocationInfo(
+      string filepath, int startline, int startcolumn, int endline, int endcolumn
+    );
+  }
+
+  /**
+   * A basic block, that is, a maximal straight-line sequence of control flow nodes
+   * without branches or joins.
+   */
+  class BasicBlock {
+    /** Gets a textual representation of this basic block. */
+    string toString();
+
+    /** Gets the enclosing callable. */
+    Callable getEnclosingCallable();
+
+    /** Gets the location of this basic block. */
+    Location getLocation();
+  }
+
+  /**
+   * Gets the basic block that immediately dominates basic block `bb`, if any.
+   *
+   * That is, all paths reaching `bb` from some entry point basic block must go
+   * through the result.
+   *
+   * Example:
+   *
+   * ```csharp
+   * int M(string s) {
+   *   if (s == null)
+   *     throw new ArgumentNullException(nameof(s));
+   *   return s.Length;
+   * }
+   * ```
+   *
+   * The basic block starting on line 2 is an immediate dominator of
+   * the basic block on line 4 (all paths from the entry point of `M`
+   * to `return s.Length;` must go through the null check.
+   */
+  BasicBlock getImmediateBasicBlockDominator(BasicBlock bb);
+
+  /** Gets an immediate successor of basic block `bb`, if any. */
+  BasicBlock getABasicBlockSuccessor(BasicBlock bb);
+
+  /** Holds if `bb` is a control-flow entry point. */
+  default predicate entryBlock(BasicBlock bb) { not exists(getImmediateBasicBlockDominator(bb)) }
+
+  /** Holds if `bb` is a control-flow exit point. */
+  default predicate exitBlock(BasicBlock bb) { not exists(getABasicBlockSuccessor(bb)) }
+
+  /** A variable that is captured in a closure. */
+  class CapturedVariable {
+    /** Gets a textual representation of this variable. */
+    string toString();
+
+    /** Gets the callable that defines this variable. */
+    Callable getCallable();
+
+    /** Gets the location of this variable. */
+    Location getLocation();
+  }
+
+  /** A parameter that is captured in a closure. */
+  class CapturedParameter extends CapturedVariable;
+
+  /**
+   * An expression with a value. That is, we expect these expressions to be
+   * represented in the data flow graph.
+   */
+  class Expr {
+    /** Gets a textual representation of this expression. */
+    string toString();
+
+    /** Gets the location of this expression. */
+    Location getLocation();
+
+    /** Holds if the `i`th node of basic block `bb` evaluates this expression. */
+    predicate hasCfgNode(BasicBlock bb, int i);
+  }
+
+  /** A write to a captured variable. */
+  class VariableWrite {
+    /** Gets the variable that is the target of this write. */
+    CapturedVariable getVariable();
+
+    /** Gets the expression that is the source of this write. */
+    Expr getSource();
+
+    /** Gets the location of this write. */
+    Location getLocation();
+
+    /** Holds if the `i`th node of basic block `bb` evaluates this expression. */
+    predicate hasCfgNode(BasicBlock bb, int i);
+  }
+
+  /** A read of a captured variable. */
+  class VariableRead extends Expr {
+    /** Gets the variable that this expression reads. */
+    CapturedVariable getVariable();
+  }
+
+  /**
+   * An expression constructing a closure that may capture one or more
+   * variables. This can for example be a lambda or a constructor call of a
+   * locally defined object.
+   */
+  class ClosureExpr extends Expr {
+    /**
+     * Holds if `body` is the callable body of this closure. A lambda expression
+     * only has one body, but in general a locally defined object may have
+     * multiple such methods and constructors.
+     */
+    predicate hasBody(Callable body);
+
+    /**
+     * Holds if `f` is an expression that may hold the value of the closure and
+     * may occur in a position where the value escapes or where the closure may
+     * be invoked.
+     *
+     * For example, if a lambda is assigned to a variable, then references to
+     * that variable in return or argument positions should be included.
+     */
+    predicate hasAliasedAccess(Expr f);
+  }
+
+  class Callable {
+    /** Gets a textual representation of this callable. */
+    string toString();
+
+    /** Gets the location of this callable. */
+    Location getLocation();
+
+    /** Holds if this callable is a constructor. */
+    predicate isConstructor();
+  }
+}
+
+signature module OutputSig<InputSig I> {
+  /**
+   * A data flow node that we need to reference in the step relations for
+   * captured variables.
+   *
+   * Note that only the `SynthesizedCaptureNode` subclass is expected to be
+   * added as additional nodes in `DataFlow::Node`. The other subclasses are
+   * expected to already be present and are included here in order to reference
+   * them in the step relations.
+   */
+  class ClosureNode;
+
+  /**
+   * A synthesized data flow node representing the storage of a captured
+   * variable.
+   */
+  class SynthesizedCaptureNode extends ClosureNode {
+    /** Gets a textual representation of this node. */
+    string toString();
+
+    /** Gets the location of this node. */
+    I::Location getLocation();
+
+    /** Gets the enclosing callable. */
+    I::Callable getEnclosingCallable();
+
+    /** Holds if this node is a synthesized access of `v`. */
+    predicate isVariableAccess(I::CapturedVariable v);
+
+    /** Holds if this node is a synthesized instance access. */
+    predicate isInstanceAccess();
+  }
+
+  /** A data flow node for an expression. */
+  class ExprNode extends ClosureNode {
+    /** Gets the expression corresponding to this node. */
+    I::Expr getExpr();
+  }
+
+  /** A data flow node for the `PostUpdateNode` of an expression. */
+  class ExprPostUpdateNode extends ClosureNode {
+    /** Gets the expression corresponding to this node. */
+    I::Expr getExpr();
+  }
+
+  /** A data flow node for a parameter. */
+  class ParameterNode extends ClosureNode {
+    /** Gets the parameter corresponding to this node. */
+    I::CapturedParameter getParameter();
+  }
+
+  /** A data flow node for an instance parameter. */
+  class ThisParameterNode extends ClosureNode {
+    /** Gets the callable this instance parameter belongs to. */
+    I::Callable getCallable();
+  }
+
+  /** A data flow node for the instance parameter argument of a constructor call. */
+  class MallocNode extends ClosureNode {
+    /** Gets the closure construction that is the post-update of this node. */
+    I::ClosureExpr getClosureExpr();
+  }
+
+  /** Holds if `post` is a `PostUpdateNode` for `pre`. */
+  predicate capturePostUpdateNode(SynthesizedCaptureNode post, SynthesizedCaptureNode pre);
+
+  /** Holds if there is a local flow step from `node1` to `node2`. */
+  predicate localFlowStep(ClosureNode node1, ClosureNode node2);
+
+  /** Holds if there is a store step from `node1` to `node2`. */
+  predicate storeStep(ClosureNode node1, I::CapturedVariable v, ClosureNode node2);
+
+  /** Holds if there is a read step from `node1` to `node2`. */
+  predicate readStep(ClosureNode node1, I::CapturedVariable v, ClosureNode node2);
+
+  /** Holds if this-to-this summaries are expected for `c`. */
+  predicate heuristicAllowInstanceParameterReturnInSelf(I::Callable c);
+}
+
+/**
+ * Constructs the type `ClosureNode` and associated step relations, which are
+ * intended to be included in the data-flow node and step relations.
+ */
+module Flow<InputSig Input> implements OutputSig<Input> {
+  private import Input
+
+  additional module ConsistencyChecks {
+    final private class FinalExpr = Expr;
+
+    private class RelevantExpr extends FinalExpr {
+      RelevantExpr() {
+        this instanceof VariableRead or
+        any(VariableWrite vw).getSource() = this or
+        this instanceof ClosureExpr or
+        any(ClosureExpr ce).hasAliasedAccess(this)
+      }
+    }
+
+    final private class FinalBasicBlock = BasicBlock;
+
+    private class RelevantBasicBlock extends FinalBasicBlock {
+      RelevantBasicBlock() {
+        exists(RelevantExpr e | e.hasCfgNode(this, _))
+        or
+        exists(VariableWrite vw | vw.hasCfgNode(this, _))
+      }
+    }
+
+    final private class FinalCallable = Callable;
+
+    private class RelevantCallable extends FinalCallable {
+      RelevantCallable() {
+        exists(RelevantBasicBlock bb | bb.getEnclosingCallable() = this)
+        or
+        exists(CapturedVariable v | v.getCallable() = this)
+        or
+        exists(ClosureExpr ce | ce.hasBody(this))
+      }
+    }
+
+    query predicate uniqueToString(string msg, int n) {
+      exists(string elem |
+        n = strictcount(RelevantBasicBlock bb | not exists(bb.toString())) and
+        elem = "BasicBlock"
+        or
+        n = strictcount(CapturedVariable v | not exists(v.toString())) and elem = "CapturedVariable"
+        or
+        n = strictcount(RelevantExpr e | not exists(e.toString())) and elem = "Expr"
+        or
+        n = strictcount(RelevantCallable c | not exists(c.toString())) and
+        elem = "Callable"
+      |
+        msg = n + " " + elem + "(s) are missing toString"
+      )
+      or
+      exists(string elem |
+        n = strictcount(RelevantBasicBlock bb | 2 <= strictcount(bb.toString())) and
+        elem = "BasicBlock"
+        or
+        n = strictcount(CapturedVariable v | 2 <= strictcount(v.toString())) and
+        elem = "CapturedVariable"
+        or
+        n = strictcount(RelevantExpr e | 2 <= strictcount(e.toString())) and
+        elem = "Expr"
+        or
+        n = strictcount(RelevantCallable c | 2 <= strictcount(c.toString())) and
+        elem = "Callable"
+      |
+        msg = n + " " + elem + "(s) have multiple toStrings"
+      )
+    }
+
+    query predicate uniqueEnclosingCallable(RelevantBasicBlock bb, string msg) {
+      msg = "BasicBlock has no enclosing callable" and not exists(bb.getEnclosingCallable())
+      or
+      msg = "BasicBlock has multiple enclosing callables" and
+      2 <= strictcount(bb.getEnclosingCallable())
+    }
+
+    query predicate uniqueDominator(RelevantBasicBlock bb, string msg) {
+      msg = "BasicBlock has multiple immediate dominators" and
+      2 <= strictcount(getImmediateBasicBlockDominator(bb))
+    }
+
+    query predicate localDominator(RelevantBasicBlock bb, string msg) {
+      msg = "BasicBlock has non-local dominator" and
+      bb.getEnclosingCallable() != getImmediateBasicBlockDominator(bb).getEnclosingCallable()
+    }
+
+    query predicate localSuccessor(RelevantBasicBlock bb, string msg) {
+      msg = "BasicBlock has non-local successor" and
+      bb.getEnclosingCallable() != getABasicBlockSuccessor(bb).getEnclosingCallable()
+    }
+
+    query predicate uniqueDefiningScope(CapturedVariable v, string msg) {
+      msg = "CapturedVariable has no defining callable" and not exists(v.getCallable())
+      or
+      msg = "CapturedVariable has multiple defining callables" and 2 <= strictcount(v.getCallable())
+    }
+
+    query predicate variableIsCaptured(CapturedVariable v, string msg) {
+      msg = "CapturedVariable is not captured" and
+      not captureAccess(v, _)
+    }
+
+    query predicate uniqueLocation(RelevantExpr e, string msg) {
+      msg = "Expr has no location" and not exists(e.getLocation())
+      or
+      msg = "Expr has multiple locations" and 2 <= strictcount(e.getLocation())
+    }
+
+    query predicate uniqueCfgNode(RelevantExpr e, string msg) {
+      msg = "Expr has no cfg node" and not e.hasCfgNode(_, _)
+      or
+      msg = "Expr has multiple cfg nodes" and
+      2 <= strictcount(BasicBlock bb, int i | e.hasCfgNode(bb, i))
+    }
+
+    private predicate uniqueWriteTarget(VariableWrite vw, string msg) {
+      msg = "VariableWrite has no target variable" and not exists(vw.getVariable())
+      or
+      msg = "VariableWrite has multiple target variables" and 2 <= strictcount(vw.getVariable())
+    }
+
+    query predicate uniqueWriteTarget(string msg) { uniqueWriteTarget(_, msg) }
+
+    private predicate uniqueWriteSource(VariableWrite vw, string msg) {
+      msg = "VariableWrite has no source expression" and not exists(vw.getSource())
+      or
+      msg = "VariableWrite has multiple source expressions" and 2 <= strictcount(vw.getSource())
+    }
+
+    query predicate uniqueWriteSource(string msg) { uniqueWriteSource(_, msg) }
+
+    private predicate uniqueWriteCfgNode(VariableWrite vw, string msg) {
+      msg = "VariableWrite has no cfg node" and not vw.hasCfgNode(_, _)
+      or
+      msg = "VariableWrite has multiple cfg nodes" and
+      2 <= strictcount(BasicBlock bb, int i | vw.hasCfgNode(bb, i))
+    }
+
+    query predicate uniqueWriteCfgNode(string msg) { uniqueWriteCfgNode(_, msg) }
+
+    private predicate localWriteStep(VariableWrite vw, string msg) {
+      exists(BasicBlock bb |
+        vw.hasCfgNode(bb, _) and
+        bb.getEnclosingCallable() != vw.getVariable().getCallable() and
+        msg = "VariableWrite is not a local step"
+      )
+    }
+
+    query predicate localWriteStep(string msg) { localWriteStep(_, msg) }
+
+    query predicate uniqueReadVariable(VariableRead vr, string msg) {
+      msg = "VariableRead has no source variable" and not exists(vr.getVariable())
+      or
+      msg = "VariableRead has multiple source variables" and 2 <= strictcount(vr.getVariable())
+    }
+
+    query predicate closureMustHaveBody(ClosureExpr ce, string msg) {
+      msg = "ClosureExpr has no body" and not ce.hasBody(_)
+    }
+
+    query predicate closureAliasMustBeLocal(ClosureExpr ce, Expr access, string msg) {
+      exists(BasicBlock bb1, BasicBlock bb2 |
+        ce.hasAliasedAccess(access) and
+        ce.hasCfgNode(bb1, _) and
+        access.hasCfgNode(bb2, _) and
+        bb1.getEnclosingCallable() != bb2.getEnclosingCallable() and
+        msg = "ClosureExpr has non-local alias - these are ignored"
+      )
+    }
+
+    private predicate astClosureParent(Callable closure, Callable parent) {
+      exists(ClosureExpr ce, BasicBlock bb |
+        ce.hasBody(closure) and ce.hasCfgNode(bb, _) and parent = bb.getEnclosingCallable()
+      )
+    }
+
+    query predicate variableAccessAstNesting(CapturedVariable v, Callable c, string msg) {
+      exists(BasicBlock bb, Callable parent |
+        captureRead(v, bb, _, false, _) or captureWrite(v, bb, _, false, _)
+      |
+        bb.getEnclosingCallable() = c and
+        v.getCallable() = parent and
+        not astClosureParent+(c, parent) and
+        msg = "CapturedVariable access is not nested in the defining callable"
+      )
+    }
+
+    query predicate uniqueCallableLocation(RelevantCallable c, string msg) {
+      msg = "Callable has no location" and not exists(c.getLocation())
+      or
+      msg = "Callable has multiple locations" and 2 <= strictcount(c.getLocation())
+    }
+
+    query predicate consistencyOverview(string msg, int n) {
+      uniqueToString(msg, n) or
+      n = strictcount(BasicBlock bb | uniqueEnclosingCallable(bb, msg)) or
+      n = strictcount(BasicBlock bb | uniqueDominator(bb, msg)) or
+      n = strictcount(BasicBlock bb | localDominator(bb, msg)) or
+      n = strictcount(BasicBlock bb | localSuccessor(bb, msg)) or
+      n = strictcount(CapturedVariable v | uniqueDefiningScope(v, msg)) or
+      n = strictcount(CapturedVariable v | variableIsCaptured(v, msg)) or
+      n = strictcount(Expr e | uniqueLocation(e, msg)) or
+      n = strictcount(Expr e | uniqueCfgNode(e, msg)) or
+      n = strictcount(VariableWrite vw | uniqueWriteTarget(vw, msg)) or
+      n = strictcount(VariableWrite vw | uniqueWriteSource(vw, msg)) or
+      n = strictcount(VariableWrite vw | uniqueWriteCfgNode(vw, msg)) or
+      n = strictcount(VariableWrite vw | localWriteStep(vw, msg)) or
+      n = strictcount(VariableRead vr | uniqueReadVariable(vr, msg)) or
+      n = strictcount(ClosureExpr ce | closureMustHaveBody(ce, msg)) or
+      n = strictcount(ClosureExpr ce, Expr access | closureAliasMustBeLocal(ce, access, msg)) or
+      n = strictcount(CapturedVariable v, Callable c | variableAccessAstNesting(v, c, msg)) or
+      n = strictcount(Callable c | uniqueCallableLocation(c, msg))
+    }
+  }
+
+  /*
+   * Flow through captured variables is handled by making each captured variable
+   * a field on the closures that capture them.
+   *
+   * For each closure creation we add a store step from the captured variable to
+   * the closure, and inside the closures we access the captured variables with
+   * a `this.` qualifier. This allows capture flow into closures.
+   *
+   * It also means that we get several aliased versions of a captured variable
+   * so proper care must be taken to be able to observe side-effects or flow out
+   * of closures. E.g. if two closures `l1` and `l2` capture `x` then we'll have
+   * three names, `x`, `l1.x`, and `l2.x`, plus any potential aliasing of the
+   * closures.
+   *
+   * To handle this, we select a primary name for a captured variable in each of
+   * its scopes, keep that name updated, and update the other names from the
+   * primary name.
+   *
+   * In the defining scope of a captured variable, we use the local variable
+   * itself as the primary storage location, and in the capturing scopes we use
+   * the synthesized field. For each relevant reference to a closure object we
+   * then update its field from the primary storage location, and we read the
+   * field back from the post-update of the closure object reference and back
+   * into the primary storage location.
+   *
+   * If we include references to a closure object that may lead to a call as
+   * relevant, then this means that we'll be able to observe the side-effects of
+   * such calls in the primary storage location.
+   *
+   * Details:
+   * For a reference to a closure `f` that captures `x` we synthesize a read of
+   * `x` at the same control-flow node. We then add a store step from `x` to `f`
+   * and a read step from `postupdate(f)` to `postupdate(x)`.
+   * ```
+   * SsaRead(x) --store[x]--> f
+   * postupdate(f) --read[x]--> postupdate(SsaRead(x))
+   * ```
+   * In a closure scope with a nested closure `g` that also captures `x` the
+   * steps instead look like this:
+   * ```
+   * SsaRead(this) --read[x]--> this.x --store[x]--> g
+   * postupdate(g) --read[x]--> postupdate(this.x)
+   * ```
+   * The final store from `postupdate(this.x)` to `postupdate(this)` is
+   * introduced automatically as a reverse read by the data flow library.
+   */
+
+  /**
+   * Holds if `vr` is a read of `v` in the `i`th node of `bb`.
+   * `topScope` is true if the read is in the defining callable of `v`.
+   */
+  private predicate captureRead(
+    CapturedVariable v, BasicBlock bb, int i, boolean topScope, VariableRead vr
+  ) {
+    vr.getVariable() = v and
+    vr.hasCfgNode(bb, i) and
+    if v.getCallable() != bb.getEnclosingCallable() then topScope = false else topScope = true
+  }
+
+  /**
+   * Holds if `vw` is a write of `v` in the `i`th node of `bb`.
+   * `topScope` is true if the write is in the defining callable of `v`.
+   */
+  private predicate captureWrite(
+    CapturedVariable v, BasicBlock bb, int i, boolean topScope, VariableWrite vw
+  ) {
+    vw.getVariable() = v and
+    vw.hasCfgNode(bb, i) and
+    if v.getCallable() != bb.getEnclosingCallable() then topScope = false else topScope = true
+  }
+
+  /** Gets the enclosing callable of `ce`. */
+  private Callable closureExprGetCallable(ClosureExpr ce) {
+    exists(BasicBlock bb | ce.hasCfgNode(bb, _) and result = bb.getEnclosingCallable())
+  }
+
+  /**
+   * Holds if `v` is available in `c` through capture. This can either be due to
+   * an explicit variable reference or through the construction of a closure
+   * that has a nested capture.
+   */
+  private predicate captureAccess(CapturedVariable v, Callable c) {
+    exists(BasicBlock bb | captureRead(v, bb, _, _, _) or captureWrite(v, bb, _, _, _) |
+      c = bb.getEnclosingCallable() and
+      c != v.getCallable()
+    )
+    or
+    exists(ClosureExpr ce |
+      c = closureExprGetCallable(ce) and
+      closureCaptures(ce, v) and
+      c != v.getCallable()
+    )
+  }
+
+  /** Holds if the closure defined by `ce` captures `v`. */
+  private predicate closureCaptures(ClosureExpr ce, CapturedVariable v) {
+    exists(Callable c | ce.hasBody(c) and captureAccess(v, c))
+  }
+
+  predicate heuristicAllowInstanceParameterReturnInSelf(Callable c) {
+    // If multiple variables are captured, then we should allow flow from one to
+    // another, which entails a this-to-this summary.
+    2 <= strictcount(CapturedVariable v | captureAccess(v, c))
+    or
+    // Constructors that capture a variable may assign it to a field, which also
+    // entails a this-to-this summary.
+    captureAccess(_, c) and c.isConstructor()
+  }
+
+  /** Holds if the constructor, if any, for the closure defined by `ce` captures `v`. */
+  private predicate hasConstructorCapture(ClosureExpr ce, CapturedVariable v) {
+    exists(Callable c | ce.hasBody(c) and c.isConstructor() and captureAccess(v, c))
+  }
+
+  /**
+   * Holds if `access` is a reference to `ce` evaluated in the `i`th node of `bb`.
+   * The reference is restricted to be in the same callable as `ce` as a
+   * precaution, even though this is expected to hold for all the given aliased
+   * accesses.
+   */
+  private predicate localClosureAccess(ClosureExpr ce, Expr access, BasicBlock bb, int i) {
+    ce.hasAliasedAccess(access) and
+    access.hasCfgNode(bb, i) and
+    pragma[only_bind_out](bb.getEnclosingCallable()) =
+      pragma[only_bind_out](closureExprGetCallable(ce))
+  }
+
+  /**
+   * Holds if we need an additional read of `v` in the `i`th node of `bb` in
+   * order to synchronize the value stored on `closure`.
+   * `topScope` is true if the read is in the defining callable of `v`.
+   *
+   * Side-effects of potentially calling `closure` at this point will be
+   * observed in a similarly synthesized post-update node for this read of `v`.
+   */
+  private predicate synthRead(
+    CapturedVariable v, BasicBlock bb, int i, boolean topScope, Expr closure
+  ) {
+    exists(ClosureExpr ce | closureCaptures(ce, v) |
+      ce.hasCfgNode(bb, i) and ce = closure
+      or
+      localClosureAccess(ce, closure, bb, i)
+    ) and
+    if v.getCallable() != bb.getEnclosingCallable() then topScope = false else topScope = true
+  }
+
+  /**
+   * Holds if there is an access of a captured variable inside a closure in the
+   * `i`th node of `bb`, such that we need to synthesize a `this.` qualifier.
+   */
+  private predicate synthThisQualifier(BasicBlock bb, int i) {
+    synthRead(_, bb, i, false, _) or
+    captureRead(_, bb, i, false, _) or
+    captureWrite(_, bb, i, false, _)
+  }
+
+  private newtype TCaptureContainer =
+    TVariable(CapturedVariable v) or
+    TThis(Callable c) { captureAccess(_, c) }
+
+  /**
+   * A storage location for a captured variable in a specific callable. This is
+   * either the variable itself (in its defining scope) or an instance variable
+   * `this` (in a capturing scope).
+   */
+  private class CaptureContainer extends TCaptureContainer {
+    string toString() {
+      exists(CapturedVariable v | this = TVariable(v) and result = v.toString())
+      or
+      result = "this" and this = TThis(_)
+    }
+  }
+
+  /** Holds if `cc` needs a definition at the entry of its callable scope. */
+  private predicate entryDef(CaptureContainer cc, BasicBlock bb, int i) {
+    exists(Callable c |
+      entryBlock(bb) and
+      pragma[only_bind_out](bb.getEnclosingCallable()) = c and
+      i =
+        min(int j |
+            j = 1 or
+            captureRead(_, bb, j, _, _) or
+            captureWrite(_, bb, j, _, _) or
+            synthRead(_, bb, j, _, _)
+          ) - 1
+    |
+      cc = TThis(c)
+      or
+      exists(CapturedParameter p | cc = TVariable(p) and p.getCallable() = c)
+    )
+  }
+
+  private module CaptureSsaInput implements Ssa::InputSig {
+    final class BasicBlock = Input::BasicBlock;
+
+    BasicBlock getImmediateBasicBlockDominator(BasicBlock bb) {
+      result = Input::getImmediateBasicBlockDominator(bb)
+    }
+
+    BasicBlock getABasicBlockSuccessor(BasicBlock bb) {
+      result = Input::getABasicBlockSuccessor(bb)
+    }
+
+    class ExitBasicBlock extends BasicBlock {
+      ExitBasicBlock() { exitBlock(this) }
+    }
+
+    class SourceVariable = CaptureContainer;
+
+    predicate variableWrite(BasicBlock bb, int i, SourceVariable cc, boolean certain) {
+      (
+        exists(CapturedVariable v | cc = TVariable(v) and captureWrite(v, bb, i, true, _))
+        or
+        entryDef(cc, bb, i)
+      ) and
+      certain = true
+    }
+
+    predicate variableRead(BasicBlock bb, int i, SourceVariable cc, boolean certain) {
+      (
+        synthThisQualifier(bb, i) and cc = TThis(bb.getEnclosingCallable())
+        or
+        exists(CapturedVariable v | cc = TVariable(v) |
+          captureRead(v, bb, i, true, _) or synthRead(v, bb, i, true, _)
+        )
+      ) and
+      certain = true
+    }
+  }
+
+  private module CaptureSsa = Ssa::Make<CaptureSsaInput>;
+
+  private newtype TClosureNode =
+    TSynthRead(CapturedVariable v, BasicBlock bb, int i, Boolean isPost) {
+      synthRead(v, bb, i, _, _)
+    } or
+    TSynthThisQualifier(BasicBlock bb, int i, Boolean isPost) { synthThisQualifier(bb, i) } or
+    TSynthPhi(CaptureSsa::DefinitionExt phi) {
+      phi instanceof CaptureSsa::PhiNode or phi instanceof CaptureSsa::PhiReadNode
+    } or
+    TExprNode(Expr expr, boolean isPost) {
+      expr instanceof VariableRead and isPost = [false, true]
+      or
+      exists(VariableWrite vw | expr = vw.getSource() and isPost = false)
+      or
+      synthRead(_, _, _, _, expr) and isPost = [false, true]
+    } or
+    TParamNode(CapturedParameter p) or
+    TThisParamNode(Callable c) { captureAccess(_, c) } or
+    TMallocNode(ClosureExpr ce) { hasConstructorCapture(ce, _) }
+
+  class ClosureNode extends TClosureNode {
+    /** Gets a textual representation of this node. */
+    string toString() {
+      exists(CapturedVariable v | this = TSynthRead(v, _, _, _) and result = v.toString())
+      or
+      result = "this" and this = TSynthThisQualifier(_, _, _)
+      or
+      exists(CaptureSsa::DefinitionExt phi, CaptureContainer cc |
+        this = TSynthPhi(phi) and
+        phi.definesAt(cc, _, _, _) and
+        result = "phi(" + cc.toString() + ")"
+      )
+      or
+      exists(Expr expr, boolean isPost | this = TExprNode(expr, isPost) |
+        isPost = false and result = expr.toString()
+        or
+        isPost = true and result = expr.toString() + " [postupdate]"
+      )
+      or
+      exists(CapturedParameter p | this = TParamNode(p) and result = p.toString())
+      or
+      result = "this" and this = TThisParamNode(_)
+      or
+      result = "malloc" and this = TMallocNode(_)
+    }
+
+    /** Gets the location of this node. */
+    Location getLocation() {
+      exists(CapturedVariable v, BasicBlock bb, int i, Expr closure |
+        this = TSynthRead(v, bb, i, _) and
+        synthRead(v, bb, i, _, closure) and
+        result = closure.getLocation()
+      )
+      or
+      exists(BasicBlock bb, int i | this = TSynthThisQualifier(bb, i, _) |
+        synthRead(_, bb, i, false, any(Expr closure | result = closure.getLocation())) or
+        captureRead(_, bb, i, false, any(VariableRead vr | result = vr.getLocation())) or
+        captureWrite(_, bb, i, false, any(VariableWrite vw | result = vw.getLocation()))
+      )
+      or
+      exists(CaptureSsa::DefinitionExt phi, BasicBlock bb |
+        this = TSynthPhi(phi) and phi.definesAt(_, bb, _, _) and result = bb.getLocation()
+      )
+      or
+      exists(Expr expr | this = TExprNode(expr, _) and result = expr.getLocation())
+      or
+      exists(CapturedParameter p | this = TParamNode(p) and result = p.getCallable().getLocation())
+      or
+      exists(Callable c | this = TThisParamNode(c) and result = c.getLocation())
+      or
+      exists(ClosureExpr ce | this = TMallocNode(ce) and result = ce.getLocation())
+    }
+  }
+
+  private class TSynthesizedCaptureNode = TSynthRead or TSynthThisQualifier or TSynthPhi;
+
+  class SynthesizedCaptureNode extends ClosureNode, TSynthesizedCaptureNode {
+    Callable getEnclosingCallable() {
+      exists(BasicBlock bb | this = TSynthRead(_, bb, _, _) and result = bb.getEnclosingCallable())
+      or
+      exists(BasicBlock bb |
+        this = TSynthThisQualifier(bb, _, _) and result = bb.getEnclosingCallable()
+      )
+      or
+      exists(CaptureSsa::DefinitionExt phi, BasicBlock bb |
+        this = TSynthPhi(phi) and phi.definesAt(_, bb, _, _) and result = bb.getEnclosingCallable()
+      )
+    }
+
+    predicate isVariableAccess(CapturedVariable v) {
+      this = TSynthRead(v, _, _, _)
+      or
+      exists(CaptureSsa::DefinitionExt phi |
+        this = TSynthPhi(phi) and phi.definesAt(TVariable(v), _, _, _)
+      )
+    }
+
+    predicate isInstanceAccess() {
+      this instanceof TSynthThisQualifier
+      or
+      exists(CaptureSsa::DefinitionExt phi |
+        this = TSynthPhi(phi) and phi.definesAt(TThis(_), _, _, _)
+      )
+    }
+  }
+
+  class ExprNode extends ClosureNode, TExprNode {
+    ExprNode() { this = TExprNode(_, false) }
+
+    Expr getExpr() { this = TExprNode(result, _) }
+  }
+
+  class ExprPostUpdateNode extends ClosureNode, TExprNode {
+    ExprPostUpdateNode() { this = TExprNode(_, true) }
+
+    Expr getExpr() { this = TExprNode(result, _) }
+  }
+
+  class ParameterNode extends ClosureNode, TParamNode {
+    CapturedParameter getParameter() { this = TParamNode(result) }
+  }
+
+  class ThisParameterNode extends ClosureNode, TThisParamNode {
+    Callable getCallable() { this = TThisParamNode(result) }
+  }
+
+  class MallocNode extends ClosureNode, TMallocNode {
+    ClosureExpr getClosureExpr() { this = TMallocNode(result) }
+  }
+
+  predicate capturePostUpdateNode(SynthesizedCaptureNode post, SynthesizedCaptureNode pre) {
+    exists(CapturedVariable v, BasicBlock bb, int i |
+      pre = TSynthRead(v, bb, i, false) and post = TSynthRead(v, bb, i, true)
+    )
+    or
+    exists(BasicBlock bb, int i |
+      pre = TSynthThisQualifier(bb, i, false) and post = TSynthThisQualifier(bb, i, true)
+    )
+  }
+
+  private predicate step(CaptureContainer cc, BasicBlock bb1, int i1, BasicBlock bb2, int i2) {
+    CaptureSsa::adjacentDefReadExt(_, cc, bb1, i1, bb2, i2)
+  }
+
+  private predicate stepToPhi(CaptureContainer cc, BasicBlock bb, int i, TSynthPhi phi) {
+    exists(CaptureSsa::DefinitionExt next |
+      CaptureSsa::lastRefRedefExt(_, cc, bb, i, next) and
+      phi = TSynthPhi(next)
+    )
+  }
+
+  private predicate ssaAccessAt(
+    ClosureNode n, CaptureContainer cc, boolean isPost, BasicBlock bb, int i
+  ) {
+    exists(CapturedVariable v |
+      synthRead(v, bb, i, true, _) and
+      n = TSynthRead(v, bb, i, isPost) and
+      cc = TVariable(v)
+    )
+    or
+    n = TSynthThisQualifier(bb, i, isPost) and cc = TThis(bb.getEnclosingCallable())
+    or
+    exists(CaptureSsa::DefinitionExt phi |
+      n = TSynthPhi(phi) and phi.definesAt(cc, bb, i, _) and isPost = false
+    )
+    or
+    exists(VariableRead vr, CapturedVariable v |
+      captureRead(v, bb, i, true, vr) and
+      n = TExprNode(vr, isPost) and
+      cc = TVariable(v)
+    )
+    or
+    exists(VariableWrite vw, CapturedVariable v |
+      captureWrite(v, bb, i, true, vw) and
+      n = TExprNode(vw.getSource(), false) and
+      isPost = false and
+      cc = TVariable(v)
+    )
+    or
+    exists(CapturedParameter p |
+      entryDef(cc, bb, i) and
+      cc = TVariable(p) and
+      n = TParamNode(p) and
+      isPost = false
+    )
+    or
+    exists(Callable c |
+      entryDef(cc, bb, i) and
+      cc = TThis(c) and
+      n = TThisParamNode(c) and
+      isPost = false
+    )
+  }
+
+  predicate localFlowStep(ClosureNode node1, ClosureNode node2) {
+    exists(CaptureContainer cc, BasicBlock bb1, int i1, BasicBlock bb2, int i2 |
+      step(cc, bb1, i1, bb2, i2) and
+      ssaAccessAt(node1, pragma[only_bind_into](cc), _, bb1, i1) and
+      ssaAccessAt(node2, pragma[only_bind_into](cc), false, bb2, i2)
+    )
+    or
+    exists(CaptureContainer cc, BasicBlock bb, int i |
+      stepToPhi(cc, bb, i, node2) and
+      ssaAccessAt(node1, cc, _, bb, i)
+    )
+  }
+
+  predicate storeStep(ClosureNode node1, CapturedVariable v, ClosureNode node2) {
+    // store v in the closure or in the malloc in case of a relevant constructor call
+    exists(BasicBlock bb, int i, Expr closure |
+      synthRead(v, bb, i, _, closure) and
+      node1 = TSynthRead(v, bb, i, false)
+    |
+      node2 = TExprNode(closure, false)
+      or
+      node2 = TMallocNode(closure) and hasConstructorCapture(closure, v)
+    )
+    or
+    // write to v inside the closure body
+    exists(BasicBlock bb, int i, VariableWrite vw |
+      captureWrite(v, bb, i, false, vw) and
+      node1 = TExprNode(vw.getSource(), false) and
+      node2 = TSynthThisQualifier(bb, i, true)
+    )
+  }
+
+  predicate readStep(ClosureNode node1, CapturedVariable v, ClosureNode node2) {
+    // read v from the closure post-update to observe side-effects
+    exists(BasicBlock bb, int i, Expr closure, boolean post |
+      synthRead(v, bb, i, _, closure) and
+      node1 = TExprNode(closure, post) and
+      node2 = TSynthRead(v, bb, i, true)
+    |
+      post = true
+      or
+      // for a constructor call the regular ExprNode is the post-update for the MallocNode
+      post = false and hasConstructorCapture(closure, v)
+    )
+    or
+    // read v from the closure inside the closure body
+    exists(BasicBlock bb, int i | node1 = TSynthThisQualifier(bb, i, false) |
+      synthRead(v, bb, i, false, _) and
+      node2 = TSynthRead(v, bb, i, false)
+      or
+      exists(VariableRead vr |
+        captureRead(v, bb, i, false, vr) and
+        node2 = TExprNode(vr, false)
+      )
+    )
+  }
+}

shared/dataflow/codeql/dataflow/internal/DataFlowImpl.qll

@@ -6,11 +6,11 @@
 
 private import codeql.util.Unit
 private import codeql.util.Option
-import DataFlowParameter
+private import codeql.dataflow.DataFlow
 
-module MakeImpl<DataFlowParameter Lang> {
+module MakeImpl<InputSig Lang> {
   private import Lang
-  private import DataFlow::DataFlowMake<Lang>
+  private import DataFlowMake<Lang>
   private import DataFlowImplCommon::MakeImplCommon<Lang>
   private import DataFlowImplCommonPublic
 

shared/dataflow/codeql/dataflow/internal/DataFlowImplCommon.qll

@@ -1,6 +1,6 @@
-import DataFlowParameter
+private import codeql.dataflow.DataFlow
 
-module MakeImplCommon<DataFlowParameter Lang> {
+module MakeImplCommon<InputSig Lang> {
   private import Lang
   import Cached
 

swift/extractor/invocation/SwiftDiagnosticsConsumer.cpp

@@ -1,5 +1,5 @@
 #include "swift/extractor/invocation/SwiftDiagnosticsConsumer.h"
-#include "swift/extractor/trap/generated/TrapClasses.h"
+#include "swift/extractor/trap/generated/TrapEntries.h"
 #include "swift/extractor/trap/TrapDomain.h"
 #include "swift/extractor/infra/SwiftDiagnosticKind.h"
 
@@ -13,17 +13,13 @@ using namespace codeql;
 
 void SwiftDiagnosticsConsumer::handleDiagnostic(swift::SourceManager& sourceManager,
                                                 const swift::DiagnosticInfo& diagInfo) {
-  if (diagInfo.IsChildNote) return;
-  Diagnostics diag{trap.createTypedLabel<DiagnosticsTag>()};
+  auto message = getDiagMessage(sourceManager, diagInfo);
+  DiagnosticsTrap diag{};
+  diag.id = trap.createTypedLabel<DiagnosticsTag>();
   diag.kind = translateDiagnosticsKind(diagInfo.Kind);
-  diag.text = getDiagMessage(sourceManager, diagInfo);
+  diag.text = message;
   trap.emit(diag);
   locationExtractor.attachLocation(sourceManager, diagInfo, diag.id);
-
-  forwardToLog(sourceManager, diagInfo, diag.text);
-  for (const auto& child : diagInfo.ChildDiagnosticInfo) {
-    forwardToLog(sourceManager, *child);
-  }
 }
 
 std::string SwiftDiagnosticsConsumer::getDiagMessage(swift::SourceManager& sourceManager,
@@ -33,29 +29,3 @@ std::string SwiftDiagnosticsConsumer::getDiagMessage(swift::SourceManager& sourc
   swift::DiagnosticEngine::formatDiagnosticText(out, diagInfo.FormatString, diagInfo.FormatArgs);
   return text.str().str();
 }
-
-void SwiftDiagnosticsConsumer::forwardToLog(swift::SourceManager& sourceManager,
-                                            const swift::DiagnosticInfo& diagInfo,
-                                            const std::string& message) {
-  auto file = sourceManager.getDisplayNameForLoc(diagInfo.Loc);
-  auto [line, column] = sourceManager.getLineAndColumnInBuffer(diagInfo.Loc);
-  using Kind = swift::DiagnosticKind;
-  switch (diagInfo.Kind) {
-    case Kind::Error:
-      LOG_ERROR("{}:{}:{} {}", file, line, column, message);
-      break;
-    case Kind::Warning:
-      LOG_WARNING("{}:{}:{} {}", file, line, column, message);
-      break;
-    case Kind::Remark:
-      LOG_INFO("{}:{}:{} {}", file, line, column, message);
-      break;
-    case Kind::Note:
-      LOG_DEBUG("{}:{}:{} {}", file, line, column, message);
-      break;
-    default:
-      LOG_ERROR("unknown diagnostic kind {}, {}:{}:{} {}", diagInfo.Kind, file, line, column,
-                message);
-      break;
-  }
-}