Merge branch 'main' of https://github.com/github/codeql

.github/workflows/ruby-qltest.yml

@@ -14,6 +14,7 @@ on:
   pull_request:
     paths:
       - "ruby/**"
+      - "shared/**"
       - .github/workflows/ruby-qltest.yml
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml

codeql-workspace.yml

@@ -4,6 +4,8 @@ provide:
   - "*/ql/test/qlpack.yml"
   - "*/ql/examples/qlpack.yml"
   - "*/ql/consistency-queries/qlpack.yml"
+  - "*/ql/automodel/src/qlpack.yml"
+  - "*/ql/automodel/test/qlpack.yml"
   - "shared/*/qlpack.yml"
   - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
   - "go/ql/config/legacy-support/qlpack.yml"

config/identical-files.json

@@ -556,5 +556,9 @@
   "EncryptionKeySizes Python/Java": [
     "python/ql/lib/semmle/python/security/internal/EncryptionKeySizes.qll",
     "java/ql/lib/semmle/code/java/security/internal/EncryptionKeySizes.qll"
+  ],
+  "Python model summaries test extension": [
+    "python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ext.yml",
+    "python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ext.yml"
   ]
 }

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj

@@ -11,12 +11,12 @@
   <ItemGroup>
     <PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
     <PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
-    <PackageReference Include="xunit" Version="2.4.2" />
+    <PackageReference Include="xunit" Version="2.5.0" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.1" />
   </ItemGroup>
 
   <ItemGroup>

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -17,7 +17,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Build" Version="17.3.2" />
+    <PackageReference Include="Microsoft.Build" Version="17.7.2" />
   </ItemGroup>
 
   <ItemGroup>

cpp/ql/lib/change-notes/2023-08-25-delete-or-delete-array.md

@@ -0,0 +1,4 @@
+---
+category: feature
+---
+* Added `DeleteOrDeleteArrayExpr` as a super type of `DeleteExpr` and `DeleteArrayExpr`

cpp/ql/lib/change-notes/2023-08-25-getAllocatorCall-deprecated.md

@@ -0,0 +1,4 @@
+---
+category: deprecated
+---
+* `getAllocatorCall` on `DeleteExpr` and `DeleteArrayExpr` has been deprecated. `getDeallocatorCall` should be used instead.

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.9.1
+version: 0.9.2-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/PrintAST.qll

@@ -826,17 +826,11 @@ private predicate namedExprChildPredicates(Expr expr, Element ele, string pred)
     or
     expr.(Conversion).getExpr() = ele and pred = "getExpr()"
     or
-    expr.(DeleteArrayExpr).getAllocatorCall() = ele and pred = "getAllocatorCall()"
+    expr.(DeleteOrDeleteArrayExpr).getDeallocatorCall() = ele and pred = "getDeallocatorCall()"
     or
-    expr.(DeleteArrayExpr).getDestructorCall() = ele and pred = "getDestructorCall()"
+    expr.(DeleteOrDeleteArrayExpr).getDestructorCall() = ele and pred = "getDestructorCall()"
     or
-    expr.(DeleteArrayExpr).getExpr() = ele and pred = "getExpr()"
+    expr.(DeleteOrDeleteArrayExpr).getExpr() = ele and pred = "getExpr()"
-    or
-    expr.(DeleteExpr).getAllocatorCall() = ele and pred = "getAllocatorCall()"
-    or
-    expr.(DeleteExpr).getDestructorCall() = ele and pred = "getDestructorCall()"
-    or
-    expr.(DeleteExpr).getExpr() = ele and pred = "getExpr()"
     or
     expr.(DestructorFieldDestruction).getExpr() = ele and pred = "getExpr()"
     or

cpp/ql/lib/semmle/code/cpp/controlflow/internal/CFG.qll

@@ -332,21 +332,12 @@ private Node getControlOrderChildSparse(Node n, int i) {
   n = any(ConditionDeclExpr cd | i = 0 and result = cd.getInitializingExpr())
   or
   n =
-    any(DeleteExpr del |
+    any(DeleteOrDeleteArrayExpr del |
       i = 0 and result = del.getExpr()
       or
       i = 1 and result = del.getDestructorCall()
       or
-      i = 2 and result = del.getAllocatorCall()
+      i = 2 and result = del.getDeallocatorCall()
-    )
-  or
-  n =
-    any(DeleteArrayExpr del |
-      i = 0 and result = del.getExpr()
-      or
-      i = 1 and result = del.getDestructorCall()
-      or
-      i = 2 and result = del.getAllocatorCall()
     )
   or
   n =

cpp/ql/lib/semmle/code/cpp/exprs/Expr.qll

@@ -932,19 +932,91 @@ class NewArrayExpr extends NewOrNewArrayExpr, @new_array_expr {
   Expr getExtent() { result = this.getChild(2) }
 }
 
+private class TDeleteOrDeleteArrayExpr = @delete_expr or @delete_array_expr;
+
+/**
+ * A C++ `delete` or `delete[]` expression.
+ */
+class DeleteOrDeleteArrayExpr extends Expr, TDeleteOrDeleteArrayExpr {
+  override int getPrecedence() { result = 16 }
+
+  /**
+   * Gets the call to a destructor that occurs prior to the object's memory being deallocated, if any.
+   *
+   * In the case of `delete[]` at runtime, the destructor will be called once for each element in the array, but the
+   * destructor call only exists once in the AST.
+   */
+  DestructorCall getDestructorCall() { result = this.getChild(1) }
+
+  /**
+   * Gets the destructor to be called to destroy the object or array, if any.
+   */
+  Destructor getDestructor() { result = this.getDestructorCall().getTarget() }
+
+  /**
+   * Gets the `operator delete` or `operator delete[]` that deallocates storage.
+   * Does not hold if the type being destroyed has a virtual destructor. In that case, the
+   * `operator delete` that will be called is determined at runtime based on the
+   * dynamic type of the object.
+   */
+  Function getDeallocator() {
+    expr_deallocator(underlyingElement(this), unresolveElement(result), _)
+  }
+
+  /**
+   * DEPRECATED: use `getDeallocatorCall` instead.
+   */
+  deprecated FunctionCall getAllocatorCall() { result = this.getChild(0) }
+
+  /**
+   * Gets the call to a non-default `operator delete`/`delete[]` that deallocates storage, if any.
+   *
+   * This will only be present when the type being deleted has a custom `operator delete` and
+   * does not have a virtual destructor.
+   */
+  FunctionCall getDeallocatorCall() { result = this.getChild(0) }
+
+  /**
+   * Holds if the deallocation function expects a size argument.
+   */
+  predicate hasSizedDeallocation() {
+    exists(int form |
+      expr_deallocator(underlyingElement(this), _, form) and
+      form.bitAnd(1) != 0 // Bit zero is the "size" bit
+    )
+  }
+
+  /**
+   * Holds if the deallocation function expects an alignment argument.
+   */
+  predicate hasAlignedDeallocation() {
+    exists(int form |
+      expr_deallocator(underlyingElement(this), _, form) and
+      form.bitAnd(2) != 0 // Bit one is the "alignment" bit
+    )
+  }
+
+  /**
+   * Gets the object or array being deleted.
+   */
+  Expr getExpr() {
+    // If there is a destructor call, the object being deleted is the qualifier
+    // otherwise it is the third child.
+    result = this.getChild(3) or result = this.getDestructorCall().getQualifier()
+  }
+}
+
 /**
  * A C++ `delete` (non-array) expression.
  * ```
  * delete ptr;
  * ```
  */
-class DeleteExpr extends Expr, @delete_expr {
+class DeleteExpr extends DeleteOrDeleteArrayExpr, @delete_expr {
   override string toString() { result = "delete" }
 
   override string getAPrimaryQlClass() { result = "DeleteExpr" }
 
-  override int getPrecedence() { result = 16 }
-
   /**
    * Gets the compile-time type of the object being deleted.
    */
@@ -957,58 +1029,6 @@ class DeleteExpr extends Expr, @delete_expr {
           .(PointerType)
           .getBaseType()
   }
-
-  /**
-   * Gets the call to a destructor that occurs prior to the object's memory being deallocated, if any.
-   */
-  DestructorCall getDestructorCall() { result = this.getChild(1) }
-
-  /**
-   * Gets the destructor to be called to destroy the object, if any.
-   */
-  Destructor getDestructor() { result = this.getDestructorCall().getTarget() }
-
-  /**
-   * Gets the `operator delete` that deallocates storage. Does not hold
-   * if the type being destroyed has a virtual destructor. In that case, the
-   * `operator delete` that will be called is determined at runtime based on the
-   * dynamic type of the object.
-   */
-  Function getDeallocator() {
-    expr_deallocator(underlyingElement(this), unresolveElement(result), _)
-  }
-
-  /**
-   * Holds if the deallocation function expects a size argument.
-   */
-  predicate hasSizedDeallocation() {
-    exists(int form |
-      expr_deallocator(underlyingElement(this), _, form) and
-      form.bitAnd(1) != 0 // Bit zero is the "size" bit
-    )
-  }
-
-  /**
-   * Holds if the deallocation function expects an alignment argument.
-   */
-  predicate hasAlignedDeallocation() {
-    exists(int form |
-      expr_deallocator(underlyingElement(this), _, form) and
-      form.bitAnd(2) != 0 // Bit one is the "alignment" bit
-    )
-  }
-
-  /**
-   * Gets the call to a non-default `operator delete` that deallocates storage, if any.
-   *
-   * This will only be present when the type being deleted has a custom `operator delete`.
-   */
-  FunctionCall getAllocatorCall() { result = this.getChild(0) }
-
-  /**
-   * Gets the object being deleted.
-   */
-  Expr getExpr() { result = this.getChild(3) or result = this.getChild(1).getChild(-1) }
 }
 
 /**
@@ -1017,13 +1037,11 @@ class DeleteExpr extends Expr, @delete_expr {
  * delete[] arr;
  * ```
  */
-class DeleteArrayExpr extends Expr, @delete_array_expr {
+class DeleteArrayExpr extends DeleteOrDeleteArrayExpr, @delete_array_expr {
   override string toString() { result = "delete[]" }
 
   override string getAPrimaryQlClass() { result = "DeleteArrayExpr" }
 
-  override int getPrecedence() { result = 16 }
-
   /**
    * Gets the element type of the array being deleted.
    */
@@ -1036,58 +1054,6 @@ class DeleteArrayExpr extends Expr, @delete_array_expr {
           .(PointerType)
           .getBaseType()
   }
-
-  /**
-   * Gets the call to a destructor that occurs prior to the array's memory being deallocated, if any.
-   *
-   * At runtime, the destructor will be called once for each element in the array, but the
-   * destructor call only exists once in the AST.
-   */
-  DestructorCall getDestructorCall() { result = this.getChild(1) }
-
-  /**
-   * Gets the destructor to be called to destroy each element in the array, if any.
-   */
-  Destructor getDestructor() { result = this.getDestructorCall().getTarget() }
-
-  /**
-   * Gets the `operator delete[]` that deallocates storage.
-   */
-  Function getDeallocator() {
-    expr_deallocator(underlyingElement(this), unresolveElement(result), _)
-  }
-
-  /**
-   * Holds if the deallocation function expects a size argument.
-   */
-  predicate hasSizedDeallocation() {
-    exists(int form |
-      expr_deallocator(underlyingElement(this), _, form) and
-      form.bitAnd(1) != 0 // Bit zero is the "size" bit
-    )
-  }
-
-  /**
-   * Holds if the deallocation function expects an alignment argument.
-   */
-  predicate hasAlignedDeallocation() {
-    exists(int form |
-      expr_deallocator(underlyingElement(this), _, form) and
-      form.bitAnd(2) != 0 // Bit one is the "alignment" bit
-    )
-  }
-
-  /**
-   * Gets the call to a non-default `operator delete` that deallocates storage, if any.
-   *
-   * This will only be present when the type being deleted has a custom `operator delete`.
-   */
-  FunctionCall getAllocatorCall() { result = this.getChild(0) }
-
-  /**
-   * Gets the array being deleted.
-   */
-  Expr getExpr() { result = this.getChild(3) or result = this.getChild(1).getChild(-1) }
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll

@@ -550,11 +550,14 @@ class SsaPhiNode extends Node, TSsaPhiNode {
    * `fromBackEdge` is true if data flows along a back-edge,
    * and `false` otherwise.
    */
+  cached
   final Node getAnInput(boolean fromBackEdge) {
     localFlowStep(result, this) and
-    if phi.getBasicBlock().dominates(result.getBasicBlock())
+    exists(IRBlock bPhi, IRBlock bResult |
-    then fromBackEdge = true
+      bPhi = phi.getBasicBlock() and bResult = result.getBasicBlock()
-    else fromBackEdge = false
+    |
+      if bPhi.dominates(bResult) then fromBackEdge = true else fromBackEdge = false
+    )
   }
 
   /** Gets a node that is used as input to this phi node. */

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/ProductFlow.qll

@@ -87,6 +87,30 @@ module ProductFlow {
      * dataflow graph.
      */
     default predicate isBarrierIn2(DataFlow::Node node) { none() }
+
+    /**
+     * Gets the virtual dispatch branching limit when calculating field flow in the first
+     * projection of the product dataflow graph.
+     *
+     * This can be overridden to a smaller value to improve performance (a
+     * value of 0 disables field flow), or a larger value to get more results.
+     */
+    default int fieldFlowBranchLimit1() {
+      // NOTE: This should be synchronized with the default value in the shared dataflow library
+      result = 2
+    }
+
+    /**
+     * Gets the virtual dispatch branching limit when calculating field flow in the second
+     * projection of the product dataflow graph.
+     *
+     * This can be overridden to a smaller value to improve performance (a
+     * value of 0 disables field flow), or a larger value to get more results.
+     */
+    default int fieldFlowBranchLimit2() {
+      // NOTE: This should be synchronized with the default value in the shared dataflow library
+      result = 2
+    }
   }
 
   /**
@@ -272,6 +296,30 @@ module ProductFlow {
      * dataflow graph.
      */
     default predicate isBarrierIn2(DataFlow::Node node) { none() }
+
+    /**
+     * Gets the virtual dispatch branching limit when calculating field flow in the first
+     * projection of the product dataflow graph.
+     *
+     * This can be overridden to a smaller value to improve performance (a
+     * value of 0 disables field flow), or a larger value to get more results.
+     */
+    default int fieldFlowBranchLimit1() {
+      // NOTE: This should be synchronized with the default value in the shared dataflow library
+      result = 2
+    }
+
+    /**
+     * Gets the virtual dispatch branching limit when calculating field flow in the second
+     * projection of the product dataflow graph.
+     *
+     * This can be overridden to a smaller value to improve performance (a
+     * value of 0 disables field flow), or a larger value to get more results.
+     */
+    default int fieldFlowBranchLimit2() {
+      // NOTE: This should be synchronized with the default value in the shared dataflow library
+      result = 2
+    }
   }
 
   /**
@@ -335,6 +383,8 @@ module ProductFlow {
       }
 
       predicate isBarrierIn(DataFlow::Node node) { Config::isBarrierIn1(node) }
+
+      int fieldFlowBranchLimit() { result = Config::fieldFlowBranchLimit1() }
     }
 
     private module Flow1 = DataFlow::GlobalWithState<Config1>;
@@ -367,6 +417,8 @@ module ProductFlow {
       }
 
       predicate isBarrierIn(DataFlow::Node node) { Config::isBarrierIn2(node) }
+
+      int fieldFlowBranchLimit() { result = Config::fieldFlowBranchLimit2() }
     }
 
     private module Flow2 = DataFlow::GlobalWithState<Config2>;

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedElement.qll

@@ -84,9 +84,9 @@ private predicate ignoreExprAndDescendants(Expr expr) {
   or
   // We do not yet translate destructors properly, so for now we ignore any
   // custom deallocator call, if present.
-  exists(DeleteExpr deleteExpr | deleteExpr.getAllocatorCall() = expr)
+  exists(DeleteExpr deleteExpr | deleteExpr.getDeallocatorCall() = expr)
   or
-  exists(DeleteArrayExpr deleteArrayExpr | deleteArrayExpr.getAllocatorCall() = expr)
+  exists(DeleteArrayExpr deleteArrayExpr | deleteArrayExpr.getDeallocatorCall() = expr)
   or
   exists(BuiltInVarArgsStart vaStartExpr |
     vaStartExpr.getLastNamedParameter().getFullyConverted() = expr

cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/RangeAnalysisUtil.qll

@@ -1,6 +1,6 @@
 /**
- * This file contains the range-analysis specific parts of the `cpp/invalid-pointer-deref` query
+ * This file contains the range-analysis specific parts of the `cpp/invalid-pointer-deref`
- * that is used by both `AllocationToInvalidPointer.qll` and `InvalidPointerToDereference.qll`.
+ * and `cpp/overrun-write` query.
  */
 
 private import cpp

cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticGuard.qll

@@ -39,6 +39,7 @@ predicate semImplies_v2(SemGuard g1, boolean b1, SemGuard g2, boolean b2) {
  * Holds if `guard` directly controls the position `controlled` with the
  * value `testIsTrue`.
  */
+pragma[nomagic]
 predicate semGuardDirectlyControlsSsaRead(
   SemGuard guard, SemSsaReadPosition controlled, boolean testIsTrue
 ) {

cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/ModulusAnalysis.qll

@@ -17,19 +17,27 @@ private import RangeUtils
 private import RangeAnalysisStage
 
 module ModulusAnalysis<DeltaSig D, BoundSig<D> Bounds, UtilSig<D> U> {
-  /**
+  pragma[nomagic]
-   * Holds if `e + delta` equals `v` at `pos`.
+  private predicate valueFlowStepSsaEqFlowCond(
-   */
+    SemSsaReadPosition pos, SemSsaVariable v, SemExpr e, int delta
-  private predicate valueFlowStepSsa(SemSsaVariable v, SemSsaReadPosition pos, SemExpr e, int delta) {
+  ) {
-    U::semSsaUpdateStep(v, e, D::fromInt(delta)) and pos.hasReadOfVar(v)
-    or
     exists(SemGuard guard, boolean testIsTrue |
-      pos.hasReadOfVar(v) and
       guard = U::semEqFlowCond(v, e, D::fromInt(delta), true, testIsTrue) and
       semGuardDirectlyControlsSsaRead(guard, pos, testIsTrue)
     )
   }
 
+  /**
+   * Holds if `e + delta` equals `v` at `pos`.
+   */
+  pragma[nomagic]
+  private predicate valueFlowStepSsa(SemSsaVariable v, SemSsaReadPosition pos, SemExpr e, int delta) {
+    U::semSsaUpdateStep(v, e, D::fromInt(delta)) and pos.hasReadOfVar(v)
+    or
+    pos.hasReadOfVar(v) and
+    valueFlowStepSsaEqFlowCond(pos, v, e, delta)
+  }
+
   /**
    * Holds if `add` is the addition of `larg` and `rarg`, neither of which are
    * `ConstantIntegerExpr`s.

cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisStage.qll

@@ -660,7 +660,7 @@ module RangeStage<
    * - `upper = false` : `v >= b + delta`
    */
   private predicate boundedSsa(
-    SemSsaVariable v, SemSsaReadPosition pos, SemBound b, D::Delta delta, boolean upper,
+    SemSsaVariable v, SemBound b, D::Delta delta, SemSsaReadPosition pos, boolean upper,
     boolean fromBackEdge, D::Delta origdelta, SemReason reason
   ) {
     exists(SemExpr mid, D::Delta d1, D::Delta d2, SemReason r1, SemReason r2 |
@@ -673,10 +673,13 @@ module RangeStage<
     )
     or
     exists(D::Delta d, SemReason r1, SemReason r2 |
-      boundedSsa(v, pos, b, d, upper, fromBackEdge, origdelta, r2) or
+      boundedSsa(pragma[only_bind_into](v), pragma[only_bind_into](b), pragma[only_bind_into](d),
-      boundedPhi(v, b, d, upper, fromBackEdge, origdelta, r2)
+        pragma[only_bind_into](pos), upper, fromBackEdge, origdelta, r2)
+      or
+      boundedPhi(pragma[only_bind_into](v), pragma[only_bind_into](b), pragma[only_bind_into](d),
+        upper, fromBackEdge, origdelta, r2)
     |
-      unequalIntegralSsa(v, pos, b, d, r1) and
+      unequalIntegralSsa(v, b, d, pos, r1) and
       (
         upper = true and delta = D::fromFloat(D::toFloat(d) - 1)
         or
@@ -694,7 +697,7 @@ module RangeStage<
    * Holds if `v != b + delta` at `pos` and `v` is of integral type.
    */
   private predicate unequalIntegralSsa(
-    SemSsaVariable v, SemSsaReadPosition pos, SemBound b, D::Delta delta, SemReason reason
+    SemSsaVariable v, SemBound b, D::Delta delta, SemSsaReadPosition pos, SemReason reason
   ) {
     exists(SemExpr e, D::Delta d1, D::Delta d2 |
       unequalFlowStepIntegralSsa(v, pos, e, d1, reason) and
@@ -746,7 +749,7 @@ module RangeStage<
   ) {
     edge.phiInput(phi, inp) and
     exists(D::Delta d, boolean fromBackEdge0 |
-      boundedSsa(inp, edge, b, d, upper, fromBackEdge0, origdelta, reason)
+      boundedSsa(inp, b, d, edge, upper, fromBackEdge0, origdelta, reason)
       or
       boundedPhi(inp, b, d, upper, fromBackEdge0, origdelta, reason)
       or
@@ -1022,7 +1025,7 @@ module RangeStage<
       reason = TSemNoReason()
       or
       exists(SemSsaVariable v, SemSsaReadPositionBlock bb |
-        boundedSsa(v, bb, b, delta, upper, fromBackEdge, origdelta, reason) and
+        boundedSsa(v, b, delta, bb, upper, fromBackEdge, origdelta, reason) and
         e = v.getAUse() and
         bb.getBlock() = e.getBasicBlock()
       )

cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeUtils.qll

@@ -49,6 +49,7 @@ module RangeUtil<Range::DeltaSig D, Range::LangSig<D> Lang> implements Range::Ut
    * - `isEq = true`  : `v == e + delta`
    * - `isEq = false` : `v != e + delta`
    */
+  pragma[nomagic]
   SemGuard semEqFlowCond(
     SemSsaVariable v, SemExpr e, D::Delta delta, boolean isEq, boolean testIsTrue
   ) {

cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/AllocationToInvalidPointer.qll

@@ -56,7 +56,7 @@ private import semmle.code.cpp.ir.dataflow.internal.ProductFlow
 private import semmle.code.cpp.ir.ValueNumbering
 private import semmle.code.cpp.controlflow.IRGuards
 private import codeql.util.Unit
-private import RangeAnalysisUtil
+private import semmle.code.cpp.rangeanalysis.new.RangeAnalysisUtil
 
 private VariableAccess getAVariableAccess(Expr e) { e.getAChild*() = result }
 
@@ -77,6 +77,15 @@ predicate hasSize(HeuristicAllocationExpr alloc, DataFlow::Node n, int state) {
   )
 }
 
+/**
+ * Gets the virtual dispatch branching limit when calculating field flow while searching
+ * for flow from an allocation to the construction of an out-of-bounds pointer.
+ *
+ * This can be overridden to a smaller value to improve performance (a
+ * value of 0 disables field flow), or a larger value to get more results.
+ */
+int allocationToInvalidPointerFieldFlowBranchLimit() { result = 0 }
+
 /**
  * A module that encapsulates a barrier guard to remove false positives from flow like:
  * ```cpp
@@ -101,9 +110,12 @@ private module SizeBarrier {
     predicate isSource(DataFlow::Node source) {
       // The sources is the same as in the sources for the second
       // projection in the `AllocToInvalidPointerConfig` module.
-      hasSize(_, source, _)
+      hasSize(_, source, _) and
+      InterestingPointerAddInstruction::isInterestingSize(source)
     }
 
+    int fieldFlowBranchLimit() { result = allocationToInvalidPointerFieldFlowBranchLimit() }
+
     /**
      * Holds if `small <= large + k` holds if `g` evaluates to `testIsTrue`.
      */
@@ -201,6 +213,8 @@ private module InterestingPointerAddInstruction {
       hasSize(source.asConvertedExpr(), _, _)
     }
 
+    int fieldFlowBranchLimit() { result = allocationToInvalidPointerFieldFlowBranchLimit() }
+
     predicate isSink(DataFlow::Node sink) {
       sink.asInstruction() = any(PointerAddInstruction pai).getLeft()
     }
@@ -220,6 +234,19 @@ private module InterestingPointerAddInstruction {
       flowTo(n)
     )
   }
+
+  /**
+   * Holds if `n` is a size of an allocation whose result flows to the left operand
+   * of a pointer-arithmetic instruction.
+   *
+   * This predicate is used to reduce the set of tuples in `SizeBarrierConfig::isSource`.
+   */
+  predicate isInterestingSize(DataFlow::Node n) {
+    exists(DataFlow::Node alloc |
+      hasSize(alloc.asConvertedExpr(), n, _) and
+      flow(alloc, _)
+    )
+  }
 }
 
 /**
@@ -244,6 +271,10 @@ private module Config implements ProductFlow::StateConfigSig {
     hasSize(allocSource.asConvertedExpr(), sizeSource, sizeAddend)
   }
 
+  int fieldFlowBranchLimit1() { result = allocationToInvalidPointerFieldFlowBranchLimit() }
+
+  int fieldFlowBranchLimit2() { result = allocationToInvalidPointerFieldFlowBranchLimit() }
+
   predicate isSinkPair(
     DataFlow::Node allocSink, FlowState1 unit, DataFlow::Node sizeSink, FlowState2 sizeAddend
   ) {

cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/InvalidPointerToDereference.qll

@@ -81,7 +81,17 @@ private import semmle.code.cpp.dataflow.new.DataFlow
 private import semmle.code.cpp.ir.ValueNumbering
 private import semmle.code.cpp.controlflow.IRGuards
 private import AllocationToInvalidPointer as AllocToInvalidPointer
-private import RangeAnalysisUtil
+private import semmle.code.cpp.rangeanalysis.new.RangeAnalysisUtil
+
+/**
+ * Gets the virtual dispatch branching limit when calculating field flow while
+ * searching for flow from an out-of-bounds pointer to a dereference of the
+ * pointer.
+ *
+ * This can be overridden to a smaller value to improve performance (a
+ * value of 0 disables field flow), or a larger value to get more results.
+ */
+int invalidPointerToDereferenceFieldFlowBranchLimit() { result = 0 }
 
 private module InvalidPointerToDerefBarrier {
   private module BarrierConfig implements DataFlow::ConfigSig {
@@ -101,6 +111,8 @@ private module InvalidPointerToDerefBarrier {
     }
 
     predicate isSink(DataFlow::Node sink) { isSink(_, sink, _, _, _) }
+
+    int fieldFlowBranchLimit() { result = invalidPointerToDereferenceFieldFlowBranchLimit() }
   }
 
   private module BarrierFlow = DataFlow::Global<BarrierConfig>;
@@ -178,6 +190,8 @@ private module InvalidPointerToDerefConfig implements DataFlow::StateConfigSig {
     // Note that this is the only place where the `FlowState` is used in this configuration.
     node = InvalidPointerToDerefBarrier::getABarrierNode(pai)
   }
+
+  int fieldFlowBranchLimit() { result = invalidPointerToDereferenceFieldFlowBranchLimit() }
 }
 
 private import DataFlow::GlobalWithState<InvalidPointerToDerefConfig>

cpp/ql/src/Likely Bugs/Likely Typos/CompareWhereAssignMeant.ql

@@ -16,6 +16,7 @@ import cpp
 from ExprInVoidContext op
 where
   not op.isUnevaluated() and
+  not inMacroExpansion(op) and
   (
     op instanceof EQExpr
     or

cpp/ql/src/Security/CWE/CWE-119/OverrunWriteProductFlow.ql

@@ -20,28 +20,10 @@ import semmle.code.cpp.models.interfaces.Allocation
 import semmle.code.cpp.models.interfaces.ArrayFunction
 import semmle.code.cpp.rangeanalysis.new.internal.semantic.analysis.RangeAnalysis
 import semmle.code.cpp.rangeanalysis.new.internal.semantic.SemanticExprSpecific
+import semmle.code.cpp.rangeanalysis.new.RangeAnalysisUtil
 import StringSizeFlow::PathGraph1
 import codeql.util.Unit
 
-pragma[nomagic]
-Instruction getABoundIn(SemBound b, IRFunction func) {
-  getSemanticExpr(result) = b.getExpr(0) and
-  result.getEnclosingIRFunction() = func
-}
-
-/**
- * Holds if `i <= b + delta`.
- */
-bindingset[i]
-pragma[inline_late]
-predicate bounded(Instruction i, Instruction b, int delta) {
-  exists(SemBound bound, IRFunction func |
-    semBounded(getSemanticExpr(i), bound, delta, true, _) and
-    b = getABoundIn(bound, func) and
-    i.getEnclosingIRFunction() = func
-  )
-}
-
 VariableAccess getAVariableAccess(Expr e) { e.getAChild*() = result }
 
 /**

cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp

@@ -3,7 +3,7 @@
   "qhelp.dtd">
 <qhelp>
 <overview>
-<p>The program performs an out-of-bounds read or write operation. In addition to causing program instability, techniques exist which may allow an attacker to use this vulnerability to execute arbitrary code.</p>
+<p>The program performs an out-of-bounds read or write operation, which can cause program instability. In addition, attackers may take advantage of the situation, and implement techniques to use this vulnerability to execute arbitrary code.</p>
 
 </overview>
 <recommendation>
@@ -13,7 +13,7 @@
 </recommendation>
 <example>
 <p>The first example allocates a buffer of size <code>size</code> and creates a local variable that stores the location that is one byte past the end of the allocation.
-This local variable is then dereferenced which results in an out-of-bounds write.
+This local variable is then dereferenced, which results in an out-of-bounds write.
 The second example subtracts one from the <code>end</code> variable before dereferencing it. This subtraction ensures that the write correctly updates the final byte of the allocation.</p>
 <sample src="InvalidPointerDeref.cpp" />
 

cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql

@@ -1,10 +1,10 @@
 /**
  * @name Invalid pointer dereference
- * @description Dereferencing a pointer that points past it allocation is undefined behavior
+ * @description Dereferencing an out-of-bounds pointer is undefined behavior and may lead to security vulnerabilities.
- *              and may lead to security vulnerabilities.
  * @kind path-problem
  * @problem.severity error
- * @precision high
+ * @security-severity 9.3
+ * @precision medium
  * @id cpp/invalid-pointer-deref
  * @tags reliability
  *       security
@@ -94,6 +94,12 @@ module FinalConfig implements DataFlow::StateConfigSig {
     )
   }
 
+  int fieldFlowBranchLimit() {
+    result =
+      allocationToInvalidPointerFieldFlowBranchLimit()
+          .maximum(invalidPointerToDereferenceFieldFlowBranchLimit())
+  }
+
   predicate isAdditionalFlowStep(
     DataFlow::Node node1, FlowState state1, DataFlow::Node node2, FlowState state2
   ) {

cpp/ql/src/Security/CWE/CWE-570/IncorrectAllocationErrorHandling.ql

@@ -17,21 +17,6 @@ import cpp
 import semmle.code.cpp.valuenumbering.GlobalValueNumbering
 import semmle.code.cpp.controlflow.Guards
 
-/**
- * A C++ `delete` or `delete[]` expression.
- */
-class DeleteOrDeleteArrayExpr extends Expr {
-  DeleteOrDeleteArrayExpr() { this instanceof DeleteExpr or this instanceof DeleteArrayExpr }
-
-  DeallocationFunction getDeallocator() {
-    result = [this.(DeleteExpr).getDeallocator(), this.(DeleteArrayExpr).getDeallocator()]
-  }
-
-  Destructor getDestructor() {
-    result = [this.(DeleteExpr).getDestructor(), this.(DeleteArrayExpr).getDestructor()]
-  }
-}
-
 /** Gets the `Constructor` invoked when `newExpr` allocates memory. */
 Constructor getConstructorForAllocation(NewOrNewArrayExpr newExpr) {
   result.getACallToThisFunction() = newExpr.getInitializer()

cpp/ql/src/change-notes/2023-08-21-invalid-pointer-deref.md

@@ -0,0 +1,4 @@
+---
+category: newQuery
+---
+* Added a new query, `cpp/invalid-pointer-deref`, to detect out-of-bounds pointer reads and writes.

cpp/ql/src/change-notes/2023-08-25-compare-where-assign-meant.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The "Comparison where assignment was intended" query (`cpp/compare-where-assign-meant`) no longer reports comparisons that appear in macro expansions.

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.7.3
+version: 0.7.4-dev
 groups: 
   - cpp
   - queries

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/InvalidPointerDeref.qlref

@@ -1 +0,0 @@
-experimental/Security/CWE/CWE-193/InvalidPointerDeref.ql

cpp/ql/test/library-tests/allocators/allocators.expected

@@ -34,11 +34,11 @@ newArrayExprDeallocators
 | allocators.cpp:108:3:108:19 | new[] | FailedInit | void FailedInit::operator delete[](void*, size_t) | 1 | 1 | sized  |
 | allocators.cpp:110:3:110:37 | new[] | FailedInitOveraligned | void FailedInitOveraligned::operator delete[](void*, std::align_val_t, float) | 128 | 128 |  aligned |
 deleteExprs
-| allocators.cpp:59:3:59:35 | delete | int | void operator delete(void*, unsigned long) | 4 | 4 | sized  |
+| allocators.cpp:59:3:59:35 | delete | int | void operator delete(void*, unsigned long) | 4 | 4 | sized  | false |
-| allocators.cpp:60:3:60:38 | delete | String | void operator delete(void*, unsigned long) | 8 | 8 | sized  |
+| allocators.cpp:60:3:60:38 | delete | String | void operator delete(void*, unsigned long) | 8 | 8 | sized  | false |
-| allocators.cpp:61:3:61:44 | delete | SizedDealloc | void SizedDealloc::operator delete(void*, size_t) | 32 | 1 | sized  |
+| allocators.cpp:61:3:61:44 | delete | SizedDealloc | void SizedDealloc::operator delete(void*, size_t) | 32 | 1 | sized  | true |
-| allocators.cpp:62:3:62:43 | delete | Overaligned | void operator delete(void*, unsigned long, std::align_val_t) | 256 | 128 | sized aligned |
+| allocators.cpp:62:3:62:43 | delete | Overaligned | void operator delete(void*, unsigned long, std::align_val_t) | 256 | 128 | sized aligned | false |
-| allocators.cpp:64:3:64:44 | delete | const String | void operator delete(void*, unsigned long) | 8 | 8 | sized  |
+| allocators.cpp:64:3:64:44 | delete | const String | void operator delete(void*, unsigned long) | 8 | 8 | sized  | false |
 deleteArrayExprs
 | allocators.cpp:78:3:78:37 | delete[] | int | void operator delete[](void*, unsigned long) | 4 | 4 | sized  |
 | allocators.cpp:79:3:79:40 | delete[] | String | void operator delete[](void*, unsigned long) | 8 | 8 | sized  |

cpp/ql/test/library-tests/allocators/allocators.ql

@@ -77,7 +77,8 @@ query predicate newArrayExprDeallocators(
 }
 
 query predicate deleteExprs(
-  DeleteExpr expr, string type, string sig, int size, int alignment, string form
+  DeleteExpr expr, string type, string sig, int size, int alignment, string form,
+  boolean hasDeallocatorCall
 ) {
   exists(Function deallocator, Type deletedType |
     expr.getDeallocator() = deallocator and
@@ -90,7 +91,10 @@ query predicate deleteExprs(
       (if expr.hasAlignedDeallocation() then aligned = "aligned" else aligned = "") and
       (if expr.hasSizedDeallocation() then sized = "sized" else sized = "") and
       form = sized + " " + aligned
-    )
+    ) and
+    if exists(expr.getDeallocatorCall())
+    then hasDeallocatorCall = true
+    else hasDeallocatorCall = false
   )
 }
 

cpp/ql/test/library-tests/ir/ir/PrintAST.expected

@@ -8477,7 +8477,7 @@ ir.cpp:
 # 1018|       getExpr(): [DeleteExpr] delete
 # 1018|           Type = [VoidType] void
 # 1018|           ValueCategory = prvalue
-# 1018|         getAllocatorCall(): [FunctionCall] call to operator delete
+# 1018|         getDeallocatorCall(): [FunctionCall] call to operator delete
 # 1018|             Type = [VoidType] void
 # 1018|             ValueCategory = prvalue
 # 1018|         getExpr(): [Literal] 0
@@ -8555,7 +8555,7 @@ ir.cpp:
 # 1027|       getExpr(): [DeleteArrayExpr] delete[]
 # 1027|           Type = [VoidType] void
 # 1027|           ValueCategory = prvalue
-# 1027|         getAllocatorCall(): [FunctionCall] call to operator delete[]
+# 1027|         getDeallocatorCall(): [FunctionCall] call to operator delete[]
 # 1027|             Type = [VoidType] void
 # 1027|             ValueCategory = prvalue
 # 1027|         getExpr(): [Literal] 0

cpp/ql/test/query-tests/Likely Bugs/Likely Typos/CompareWhereAssignMeant/CompareWhereAssignMeant.expected

@@ -6,3 +6,5 @@
 | test.cpp:39:23:39:28 | ... == ... | This '==' operator has no effect. The assignment ('=') operator was probably intended. |
 | test.cpp:42:23:42:28 | ... == ... | This '==' operator has no effect. The assignment ('=') operator was probably intended. |
 | test.cpp:51:13:51:13 | call to operator== | This '==' operator has no effect. The assignment ('=') operator was probably intended. |
+| test.cpp:72:3:72:8 | ... == ... | This '==' operator has no effect. The assignment ('=') operator was probably intended. |
+| test.cpp:73:3:73:12 | ... == ... | This '==' operator has no effect. The assignment ('=') operator was probably intended. |

cpp/ql/test/query-tests/Likely Bugs/Likely Typos/CompareWhereAssignMeant/test.cpp

@@ -61,3 +61,14 @@ template<typename T1, typename T2>
 auto sfinaeTrick(T1 x1, T2 x2) -> decltype(x1 == x2, bool()) { // GOOD
   return x1 == x2;
 }
+
+void report_error(const char*);
+
+#define DOES_NOT_THROW(E) do { try { E; } catch (...) { report_error(""); } } while(0)
+#define ID(X) (X)
+
+void test_inside_macro_expansion(int x, int y) {
+  DOES_NOT_THROW(x == y); // GOOD
+  x == y; // BAD
+  x == ID(y); // BAD
+}

cpp/ql/test/query-tests/Security/CWE/CWE-193/AllocationToInvalidPointer.expected

@@ -1,2 +1,2 @@
-failures
 testFailures
+failures

cpp/ql/test/query-tests/Security/CWE/CWE-193/InvalidPointerDeref.expected

@@ -45,58 +45,6 @@ edges
 | test.cpp:53:5:53:23 | ... = ... | test.cpp:51:33:51:35 | end |
 | test.cpp:53:12:53:23 | ... + ... | test.cpp:53:5:53:23 | ... = ... |
 | test.cpp:60:34:60:37 | mk_array output argument | test.cpp:67:9:67:14 | ... = ... |
-| test.cpp:80:9:80:16 | mk_array indirection [end] | test.cpp:89:19:89:26 | call to mk_array [end] |
-| test.cpp:80:9:80:16 | mk_array indirection [end] | test.cpp:119:18:119:25 | call to mk_array [end] |
-| test.cpp:82:5:82:28 | ... = ... | test.cpp:82:9:82:13 | arr indirection [post update] [begin] |
-| test.cpp:82:9:82:13 | arr indirection [post update] [begin] | test.cpp:83:15:83:17 | arr indirection [begin] |
-| test.cpp:82:17:82:22 | call to malloc | test.cpp:82:5:82:28 | ... = ... |
-| test.cpp:83:5:83:30 | ... = ... | test.cpp:83:9:83:11 | arr indirection [post update] [end] |
-| test.cpp:83:9:83:11 | arr indirection [post update] [end] | test.cpp:80:9:80:16 | mk_array indirection [end] |
-| test.cpp:83:15:83:17 | arr indirection [begin] | test.cpp:83:19:83:23 | begin indirection |
-| test.cpp:83:15:83:30 | ... + ... | test.cpp:83:5:83:30 | ... = ... |
-| test.cpp:83:19:83:23 | begin indirection | test.cpp:83:5:83:30 | ... = ... |
-| test.cpp:83:19:83:23 | begin indirection | test.cpp:83:15:83:30 | ... + ... |
-| test.cpp:89:19:89:26 | call to mk_array [end] | test.cpp:91:36:91:38 | arr indirection [end] |
-| test.cpp:89:19:89:26 | call to mk_array [end] | test.cpp:95:36:95:38 | arr indirection [end] |
-| test.cpp:91:36:91:38 | arr indirection [end] | test.cpp:91:40:91:42 | end indirection |
-| test.cpp:91:36:91:38 | arr indirection [end] | test.cpp:96:9:96:14 | ... = ... |
-| test.cpp:91:40:91:42 | end indirection | test.cpp:96:9:96:14 | ... = ... |
-| test.cpp:95:36:95:38 | arr indirection [end] | test.cpp:95:40:95:42 | end indirection |
-| test.cpp:95:36:95:38 | arr indirection [end] | test.cpp:96:9:96:14 | ... = ... |
-| test.cpp:95:40:95:42 | end indirection | test.cpp:96:9:96:14 | ... = ... |
-| test.cpp:104:27:104:29 | arr [end] | test.cpp:105:36:105:38 | arr indirection [end] |
-| test.cpp:104:27:104:29 | arr [end] | test.cpp:109:36:109:38 | arr indirection [end] |
-| test.cpp:105:36:105:38 | arr indirection [end] | test.cpp:105:40:105:42 | end indirection |
-| test.cpp:105:36:105:38 | arr indirection [end] | test.cpp:110:9:110:14 | ... = ... |
-| test.cpp:105:40:105:42 | end indirection | test.cpp:110:9:110:14 | ... = ... |
-| test.cpp:109:36:109:38 | arr indirection [end] | test.cpp:109:40:109:42 | end indirection |
-| test.cpp:109:36:109:38 | arr indirection [end] | test.cpp:110:9:110:14 | ... = ... |
-| test.cpp:109:40:109:42 | end indirection | test.cpp:110:9:110:14 | ... = ... |
-| test.cpp:119:18:119:25 | call to mk_array [end] | test.cpp:104:27:104:29 | arr [end] |
-| test.cpp:141:10:141:19 | mk_array_p indirection [end] | test.cpp:150:20:150:29 | call to mk_array_p indirection [end] |
-| test.cpp:141:10:141:19 | mk_array_p indirection [end] | test.cpp:180:19:180:28 | call to mk_array_p indirection [end] |
-| test.cpp:143:5:143:29 | ... = ... | test.cpp:143:10:143:14 | arr indirection [post update] [begin] |
-| test.cpp:143:10:143:14 | arr indirection [post update] [begin] | test.cpp:144:16:144:18 | arr indirection [begin] |
-| test.cpp:143:18:143:23 | call to malloc | test.cpp:143:5:143:29 | ... = ... |
-| test.cpp:144:5:144:32 | ... = ... | test.cpp:144:10:144:12 | arr indirection [post update] [end] |
-| test.cpp:144:10:144:12 | arr indirection [post update] [end] | test.cpp:141:10:141:19 | mk_array_p indirection [end] |
-| test.cpp:144:16:144:18 | arr indirection [begin] | test.cpp:144:21:144:25 | begin indirection |
-| test.cpp:144:16:144:32 | ... + ... | test.cpp:144:5:144:32 | ... = ... |
-| test.cpp:144:21:144:25 | begin indirection | test.cpp:144:5:144:32 | ... = ... |
-| test.cpp:144:21:144:25 | begin indirection | test.cpp:144:16:144:32 | ... + ... |
-| test.cpp:150:20:150:29 | call to mk_array_p indirection [end] | test.cpp:156:37:156:39 | arr indirection [end] |
-| test.cpp:156:37:156:39 | arr indirection [end] | test.cpp:156:42:156:44 | end indirection |
-| test.cpp:156:37:156:39 | arr indirection [end] | test.cpp:157:9:157:14 | ... = ... |
-| test.cpp:156:42:156:44 | end indirection | test.cpp:157:9:157:14 | ... = ... |
-| test.cpp:165:29:165:31 | arr indirection [end] | test.cpp:166:37:166:39 | arr indirection [end] |
-| test.cpp:165:29:165:31 | arr indirection [end] | test.cpp:170:37:170:39 | arr indirection [end] |
-| test.cpp:166:37:166:39 | arr indirection [end] | test.cpp:166:42:166:44 | end indirection |
-| test.cpp:166:37:166:39 | arr indirection [end] | test.cpp:171:9:171:14 | ... = ... |
-| test.cpp:166:42:166:44 | end indirection | test.cpp:171:9:171:14 | ... = ... |
-| test.cpp:170:37:170:39 | arr indirection [end] | test.cpp:170:42:170:44 | end indirection |
-| test.cpp:170:37:170:39 | arr indirection [end] | test.cpp:171:9:171:14 | ... = ... |
-| test.cpp:170:42:170:44 | end indirection | test.cpp:171:9:171:14 | ... = ... |
-| test.cpp:180:19:180:28 | call to mk_array_p indirection [end] | test.cpp:165:29:165:31 | arr indirection [end] |
 | test.cpp:194:23:194:28 | call to malloc | test.cpp:195:17:195:23 | ... + ... |
 | test.cpp:194:23:194:28 | call to malloc | test.cpp:195:17:195:23 | ... + ... |
 | test.cpp:194:23:194:28 | call to malloc | test.cpp:201:5:201:19 | ... = ... |
@@ -233,6 +181,18 @@ edges
 | test.cpp:754:18:754:31 | new[] | test.cpp:772:16:772:29 | access to array |
 | test.cpp:754:18:754:31 | new[] | test.cpp:772:16:772:29 | access to array |
 | test.cpp:781:14:781:27 | new[] | test.cpp:786:18:786:27 | access to array |
+| test.cpp:792:60:792:62 | end | test.cpp:800:40:800:43 | mk_array_no_field_flow output argument |
+| test.cpp:792:60:792:62 | end | test.cpp:832:40:832:43 | mk_array_no_field_flow output argument |
+| test.cpp:793:14:793:19 | call to malloc | test.cpp:794:5:794:24 | ... = ... |
+| test.cpp:793:14:793:19 | call to malloc | test.cpp:794:12:794:24 | ... + ... |
+| test.cpp:794:5:794:24 | ... = ... | test.cpp:792:60:792:62 | end |
+| test.cpp:794:12:794:24 | ... + ... | test.cpp:794:5:794:24 | ... = ... |
+| test.cpp:800:40:800:43 | mk_array_no_field_flow output argument | test.cpp:807:7:807:12 | ... = ... |
+| test.cpp:815:52:815:54 | end | test.cpp:815:52:815:54 | end |
+| test.cpp:815:52:815:54 | end | test.cpp:821:7:821:12 | ... = ... |
+| test.cpp:815:52:815:54 | end | test.cpp:821:7:821:12 | ... = ... |
+| test.cpp:832:40:832:43 | mk_array_no_field_flow output argument | test.cpp:833:37:833:39 | end |
+| test.cpp:833:37:833:39 | end | test.cpp:815:52:815:54 | end |
 nodes
 | test.cpp:4:15:4:20 | call to malloc | semmle.label | call to malloc |
 | test.cpp:5:15:5:22 | ... + ... | semmle.label | ... + ... |
@@ -260,48 +220,6 @@ nodes
 | test.cpp:53:12:53:23 | ... + ... | semmle.label | ... + ... |
 | test.cpp:60:34:60:37 | mk_array output argument | semmle.label | mk_array output argument |
 | test.cpp:67:9:67:14 | ... = ... | semmle.label | ... = ... |
-| test.cpp:80:9:80:16 | mk_array indirection [end] | semmle.label | mk_array indirection [end] |
-| test.cpp:82:5:82:28 | ... = ... | semmle.label | ... = ... |
-| test.cpp:82:9:82:13 | arr indirection [post update] [begin] | semmle.label | arr indirection [post update] [begin] |
-| test.cpp:82:17:82:22 | call to malloc | semmle.label | call to malloc |
-| test.cpp:83:5:83:30 | ... = ... | semmle.label | ... = ... |
-| test.cpp:83:9:83:11 | arr indirection [post update] [end] | semmle.label | arr indirection [post update] [end] |
-| test.cpp:83:15:83:17 | arr indirection [begin] | semmle.label | arr indirection [begin] |
-| test.cpp:83:15:83:30 | ... + ... | semmle.label | ... + ... |
-| test.cpp:83:19:83:23 | begin indirection | semmle.label | begin indirection |
-| test.cpp:89:19:89:26 | call to mk_array [end] | semmle.label | call to mk_array [end] |
-| test.cpp:91:36:91:38 | arr indirection [end] | semmle.label | arr indirection [end] |
-| test.cpp:91:40:91:42 | end indirection | semmle.label | end indirection |
-| test.cpp:95:36:95:38 | arr indirection [end] | semmle.label | arr indirection [end] |
-| test.cpp:95:40:95:42 | end indirection | semmle.label | end indirection |
-| test.cpp:96:9:96:14 | ... = ... | semmle.label | ... = ... |
-| test.cpp:104:27:104:29 | arr [end] | semmle.label | arr [end] |
-| test.cpp:105:36:105:38 | arr indirection [end] | semmle.label | arr indirection [end] |
-| test.cpp:105:40:105:42 | end indirection | semmle.label | end indirection |
-| test.cpp:109:36:109:38 | arr indirection [end] | semmle.label | arr indirection [end] |
-| test.cpp:109:40:109:42 | end indirection | semmle.label | end indirection |
-| test.cpp:110:9:110:14 | ... = ... | semmle.label | ... = ... |
-| test.cpp:119:18:119:25 | call to mk_array [end] | semmle.label | call to mk_array [end] |
-| test.cpp:141:10:141:19 | mk_array_p indirection [end] | semmle.label | mk_array_p indirection [end] |
-| test.cpp:143:5:143:29 | ... = ... | semmle.label | ... = ... |
-| test.cpp:143:10:143:14 | arr indirection [post update] [begin] | semmle.label | arr indirection [post update] [begin] |
-| test.cpp:143:18:143:23 | call to malloc | semmle.label | call to malloc |
-| test.cpp:144:5:144:32 | ... = ... | semmle.label | ... = ... |
-| test.cpp:144:10:144:12 | arr indirection [post update] [end] | semmle.label | arr indirection [post update] [end] |
-| test.cpp:144:16:144:18 | arr indirection [begin] | semmle.label | arr indirection [begin] |
-| test.cpp:144:16:144:32 | ... + ... | semmle.label | ... + ... |
-| test.cpp:144:21:144:25 | begin indirection | semmle.label | begin indirection |
-| test.cpp:150:20:150:29 | call to mk_array_p indirection [end] | semmle.label | call to mk_array_p indirection [end] |
-| test.cpp:156:37:156:39 | arr indirection [end] | semmle.label | arr indirection [end] |
-| test.cpp:156:42:156:44 | end indirection | semmle.label | end indirection |
-| test.cpp:157:9:157:14 | ... = ... | semmle.label | ... = ... |
-| test.cpp:165:29:165:31 | arr indirection [end] | semmle.label | arr indirection [end] |
-| test.cpp:166:37:166:39 | arr indirection [end] | semmle.label | arr indirection [end] |
-| test.cpp:166:42:166:44 | end indirection | semmle.label | end indirection |
-| test.cpp:170:37:170:39 | arr indirection [end] | semmle.label | arr indirection [end] |
-| test.cpp:170:42:170:44 | end indirection | semmle.label | end indirection |
-| test.cpp:171:9:171:14 | ... = ... | semmle.label | ... = ... |
-| test.cpp:180:19:180:28 | call to mk_array_p indirection [end] | semmle.label | call to mk_array_p indirection [end] |
 | test.cpp:194:23:194:28 | call to malloc | semmle.label | call to malloc |
 | test.cpp:195:17:195:23 | ... + ... | semmle.label | ... + ... |
 | test.cpp:195:17:195:23 | ... + ... | semmle.label | ... + ... |
@@ -394,6 +312,17 @@ nodes
 | test.cpp:772:16:772:29 | access to array | semmle.label | access to array |
 | test.cpp:781:14:781:27 | new[] | semmle.label | new[] |
 | test.cpp:786:18:786:27 | access to array | semmle.label | access to array |
+| test.cpp:792:60:792:62 | end | semmle.label | end |
+| test.cpp:793:14:793:19 | call to malloc | semmle.label | call to malloc |
+| test.cpp:794:5:794:24 | ... = ... | semmle.label | ... = ... |
+| test.cpp:794:12:794:24 | ... + ... | semmle.label | ... + ... |
+| test.cpp:800:40:800:43 | mk_array_no_field_flow output argument | semmle.label | mk_array_no_field_flow output argument |
+| test.cpp:807:7:807:12 | ... = ... | semmle.label | ... = ... |
+| test.cpp:815:52:815:54 | end | semmle.label | end |
+| test.cpp:815:52:815:54 | end | semmle.label | end |
+| test.cpp:821:7:821:12 | ... = ... | semmle.label | ... = ... |
+| test.cpp:832:40:832:43 | mk_array_no_field_flow output argument | semmle.label | mk_array_no_field_flow output argument |
+| test.cpp:833:37:833:39 | end | semmle.label | end |
 subpaths
 #select
 | test.cpp:6:14:6:15 | * ... | test.cpp:4:15:4:20 | call to malloc | test.cpp:6:14:6:15 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:4:15:4:20 | call to malloc | call to malloc | test.cpp:5:19:5:22 | size | size |
@@ -404,10 +333,6 @@ subpaths
 | test.cpp:42:14:42:15 | * ... | test.cpp:40:15:40:20 | call to malloc | test.cpp:42:14:42:15 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:40:15:40:20 | call to malloc | call to malloc | test.cpp:41:20:41:27 | ... - ... | ... - ... |
 | test.cpp:44:14:44:21 | * ... | test.cpp:40:15:40:20 | call to malloc | test.cpp:44:14:44:21 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@ + 1. | test.cpp:40:15:40:20 | call to malloc | call to malloc | test.cpp:41:20:41:27 | ... - ... | ... - ... |
 | test.cpp:67:9:67:14 | ... = ... | test.cpp:52:19:52:24 | call to malloc | test.cpp:67:9:67:14 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:52:19:52:24 | call to malloc | call to malloc | test.cpp:53:20:53:23 | size | size |
-| test.cpp:96:9:96:14 | ... = ... | test.cpp:82:17:82:22 | call to malloc | test.cpp:96:9:96:14 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:82:17:82:22 | call to malloc | call to malloc | test.cpp:83:27:83:30 | size | size |
-| test.cpp:110:9:110:14 | ... = ... | test.cpp:82:17:82:22 | call to malloc | test.cpp:110:9:110:14 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:82:17:82:22 | call to malloc | call to malloc | test.cpp:83:27:83:30 | size | size |
-| test.cpp:157:9:157:14 | ... = ... | test.cpp:143:18:143:23 | call to malloc | test.cpp:157:9:157:14 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:143:18:143:23 | call to malloc | call to malloc | test.cpp:144:29:144:32 | size | size |
-| test.cpp:171:9:171:14 | ... = ... | test.cpp:143:18:143:23 | call to malloc | test.cpp:171:9:171:14 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:143:18:143:23 | call to malloc | call to malloc | test.cpp:144:29:144:32 | size | size |
 | test.cpp:201:5:201:19 | ... = ... | test.cpp:194:23:194:28 | call to malloc | test.cpp:201:5:201:19 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:194:23:194:28 | call to malloc | call to malloc | test.cpp:195:21:195:23 | len | len |
 | test.cpp:213:5:213:13 | ... = ... | test.cpp:205:23:205:28 | call to malloc | test.cpp:213:5:213:13 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:205:23:205:28 | call to malloc | call to malloc | test.cpp:206:21:206:23 | len | len |
 | test.cpp:232:3:232:20 | ... = ... | test.cpp:231:18:231:30 | new[] | test.cpp:232:3:232:20 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:231:18:231:30 | new[] | new[] | test.cpp:232:11:232:15 | index | index |
@@ -434,3 +359,5 @@ subpaths
 | test.cpp:772:16:772:29 | access to array | test.cpp:754:18:754:31 | new[] | test.cpp:772:16:772:29 | access to array | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:754:18:754:31 | new[] | new[] | test.cpp:767:22:767:28 | ... + ... | ... + ... |
 | test.cpp:772:16:772:29 | access to array | test.cpp:754:18:754:31 | new[] | test.cpp:772:16:772:29 | access to array | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:754:18:754:31 | new[] | new[] | test.cpp:772:22:772:28 | ... + ... | ... + ... |
 | test.cpp:786:18:786:27 | access to array | test.cpp:781:14:781:27 | new[] | test.cpp:786:18:786:27 | access to array | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:781:14:781:27 | new[] | new[] | test.cpp:786:20:786:26 | ... + ... | ... + ... |
+| test.cpp:807:7:807:12 | ... = ... | test.cpp:793:14:793:19 | call to malloc | test.cpp:807:7:807:12 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:793:14:793:19 | call to malloc | call to malloc | test.cpp:794:21:794:24 | size | size |
+| test.cpp:821:7:821:12 | ... = ... | test.cpp:793:14:793:19 | call to malloc | test.cpp:821:7:821:12 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:793:14:793:19 | call to malloc | call to malloc | test.cpp:794:21:794:24 | size | size |

cpp/ql/test/query-tests/Security/CWE/CWE-193/InvalidPointerDeref.qlref

@@ -0,0 +1 @@
+Security/CWE/CWE-193/InvalidPointerDeref.ql

cpp/ql/test/query-tests/Security/CWE/CWE-193/InvalidPointerToDereference.expected

@@ -1,2 +1,2 @@
-failures
 testFailures
+failures

cpp/ql/test/query-tests/Security/CWE/CWE-193/test.cpp

@@ -80,7 +80,7 @@ struct array_t {
 array_t mk_array(int size) {
     array_t arr;
     arr.begin = malloc(size);
-    arr.end = arr.begin + size; // $ alloc=L82
+    arr.end = arr.begin + size; // $ MISSING: alloc=L82
 
     return arr;
 }
@@ -93,7 +93,7 @@ void test6(int size) {
     }
 
     for (char* p = arr.begin; p <= arr.end; ++p) {
-        *p = 0; // $ deref=L83->L91->L96 deref=L83->L95->L96 // BAD
+        *p = 0; // $ MISSING: deref=L83->L91->L96 deref=L83->L95->L96 // BAD [NOT DETECTED]
     }
 
     for (char* p = arr.begin; p < arr.end; ++p) {
@@ -107,7 +107,7 @@ void test7_callee(array_t arr) {
     }
 
     for (char* p = arr.begin; p <= arr.end; ++p) {
-        *p = 0; // $ deref=L83->L105->L110 deref=L83->L109->L110 // BAD
+        *p = 0; // $ MISSING: deref=L83->L105->L110 deref=L83->L109->L110 // BAD [NOT DETECTED]
     }
 
     for (char* p = arr.begin; p < arr.end; ++p) {
@@ -141,7 +141,7 @@ void test8(int size) {
 array_t *mk_array_p(int size) {
     array_t *arr = (array_t*) malloc(sizeof(array_t));
     arr->begin = malloc(size);
-    arr->end = arr->begin + size; // $ alloc=L143
+    arr->end = arr->begin + size; // $ MISSING: alloc=L143
 
     return arr;
 }
@@ -154,7 +154,7 @@ void test9(int size) {
     }
 
     for (char* p = arr->begin; p <= arr->end; ++p) {
-        *p = 0; // $ deref=L144->L156->L157 // BAD
+        *p = 0; // $ MISSING: deref=L144->L156->L157 // BAD [NOT DETECTED]
     }
 
     for (char* p = arr->begin; p < arr->end; ++p) {
@@ -168,7 +168,7 @@ void test10_callee(array_t *arr) {
     }
 
     for (char* p = arr->begin; p <= arr->end; ++p) {
-        *p = 0; // $ deref=L144->L166->L171 deref=L144->L170->L171 // BAD
+        *p = 0; // $ MISSING: deref=L144->L166->L171 deref=L144->L170->L171 // BAD [NOT DETECTED]
     }
 
     for (char* p = arr->begin; p < arr->end; ++p) {
@@ -788,3 +788,47 @@ void test38_simple(unsigned size, unsigned pos, unsigned numParams) {
     }
   }
 }
+
+void mk_array_no_field_flow(int size, char** begin, char** end) {
+    *begin = malloc(size);
+    *end = *begin + size; // $ alloc=L793
+}
+
+void test6_no_field_flow(int size) {
+  char* begin;
+  char* end;
+  mk_array_no_field_flow(size, &begin, &end);
+
+  for (char* p = begin; p != end; ++p) {
+      *p = 0; // GOOD
+  }
+
+  for (char* p = begin; p <= end; ++p) {
+      *p = 0; // $ deref=L794->L802->L807 deref=L794->L806->L807 // BAD
+  }
+
+  for (char* p = begin; p < end; ++p) {
+      *p = 0; // GOOD
+  }
+}
+
+void test7_callee_no_field_flow(char* begin, char* end) {
+  for (char* p = begin; p != end; ++p) {
+      *p = 0; // GOOD
+  }
+
+  for (char* p = begin; p <= end; ++p) {
+      *p = 0; // $ deref=L794->L815->L821 deref=L794->L816->L821 deref=L794->L820->L821 // BAD
+  }
+
+  for (char* p = begin; p < end; ++p) {
+      *p = 0; // GOOD
+  }
+}
+
+void test7_no_field_flow(int size) {
+  char* begin;
+  char* end;
+  mk_array_no_field_flow(size, &begin, &end);
+  test7_callee_no_field_flow(begin, end);
+}

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -1,12 +1,12 @@
 using Xunit;
-using Semmle.Autobuild.Shared;
-using Semmle.Util;
-using System.Collections.Generic;
 using System;
-using System.Linq;
+using System.Collections.Generic;
-using Microsoft.Build.Construction;
-using System.Xml;
 using System.IO;
+using System.Linq;
+using System.Xml;
+using Microsoft.Build.Construction;
+using Semmle.Util;
+using Semmle.Autobuild.Shared;
 
 namespace Semmle.Autobuild.CSharp.Tests
 {

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/Semmle.Autobuild.CSharp.Tests.csproj

@@ -8,12 +8,12 @@
 	<ItemGroup>
 		<PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
 		<PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
-		<PackageReference Include="xunit" Version="2.4.2" />
+		<PackageReference Include="xunit" Version="2.5.0" />
-		<PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+		<PackageReference Include="xunit.runner.visualstudio" Version="2.5.0">
 			<PrivateAssets>all</PrivateAssets>
 			<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
 		</PackageReference>
-		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
+		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.1" />
 	</ItemGroup>
 	<ItemGroup>
 		<ProjectReference Include="..\Semmle.Autobuild.CSharp\Semmle.Autobuild.CSharp.csproj" />

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

@@ -1,8 +1,8 @@
-using Semmle.Extraction.CSharp;
+using System.Linq;
+using Semmle.Util;
 using Semmle.Util.Logging;
 using Semmle.Autobuild.Shared;
-using Semmle.Util;
+using Semmle.Extraction.CSharp;
-using System.Linq;
 
 namespace Semmle.Autobuild.CSharp
 {

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpDiagnosticClassifier.cs

@@ -2,8 +2,8 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
-using Semmle.Autobuild.Shared;
 using Semmle.Util;
+using Semmle.Autobuild.Shared;
 
 namespace Semmle.Autobuild.CSharp
 {

csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs

@@ -1,11 +1,11 @@
 using System;
-using Semmle.Util.Logging;
-using System.Linq;
-using Newtonsoft.Json.Linq;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using Semmle.Util;
+using Semmle.Util.Logging;
 using Semmle.Autobuild.Shared;
+using Newtonsoft.Json.Linq;
 
 namespace Semmle.Autobuild.CSharp
 {

csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

@@ -14,8 +14,8 @@
 		<Folder Include="Properties\" />
 	</ItemGroup>
 	<ItemGroup>
-		<PackageReference Include="Microsoft.Build" Version="17.3.2" />
+		<PackageReference Include="Microsoft.Build" Version="17.7.2" />
-		<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
+		<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
 	</ItemGroup>
 	<ItemGroup>
 		<ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />

csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs

@@ -1,9 +1,9 @@
-using Semmle.Util;
-using Semmle.Util.Logging;
 using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using Semmle.Util;
+using Semmle.Util.Logging;
 
 namespace Semmle.Autobuild.Shared
 {

csharp/autobuilder/Semmle.Autobuild.Shared/BuildActions.cs

@@ -1,13 +1,11 @@
-using Semmle.Util;
+using System;
-using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.IO;
-using System.Xml;
-using System.Net.Http;
 using System.Diagnostics.CodeAnalysis;
-using System.Threading.Tasks;
+using System.IO;
 using System.Runtime.InteropServices;
+using System.Xml;
+using Semmle.Util;
 
 namespace Semmle.Autobuild.Shared
 {
@@ -283,17 +281,8 @@ namespace Semmle.Autobuild.Shared
 
         public string EnvironmentExpandEnvironmentVariables(string s) => Environment.ExpandEnvironmentVariables(s);
 
-        private static async Task DownloadFileAsync(string address, string filename)
-        {
-            using var httpClient = new HttpClient();
-            using var request = new HttpRequestMessage(HttpMethod.Get, address);
-            using var contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync();
-            using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
-            await contentStream.CopyToAsync(stream);
-        }
-
         public void DownloadFile(string address, string fileName) =>
-            DownloadFileAsync(address, fileName).Wait();
+            FileUtils.DownloadFile(address, fileName);
 
         public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new DiagnosticsStream(filename);
 

csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs

@@ -1,6 +1,7 @@
-using Semmle.Util.Logging;
 using System.Collections.Generic;
 using System.Linq;
+using Semmle.Util;
+using Semmle.Util.Logging;
 
 namespace Semmle.Autobuild.Shared
 {
@@ -190,7 +191,7 @@ namespace Semmle.Autobuild.Shared
             })
             &
             BuildScript.DownloadFile(
-                "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe",
+                FileUtils.NugetExeUrl,
                 path,
                 e => builder.Log(Severity.Warning, $"Failed to download 'nuget.exe': {e.Message}"))
             &

csharp/autobuilder/Semmle.Autobuild.Shared/Semmle.Autobuild.Shared.csproj

@@ -11,7 +11,7 @@
 		<Folder Include="Properties\" />
 	</ItemGroup>
 	<ItemGroup>
-		<PackageReference Include="Microsoft.Build" Version="17.3.2" />
+		<PackageReference Include="Microsoft.Build" Version="17.7.2" />
 	</ItemGroup>
 	<ItemGroup>
 		<ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />

csharp/autobuilder/Semmle.Autobuild.Shared/Solution.cs

@@ -1,9 +1,9 @@
-using Microsoft.Build.Construction;
+using System;
-using Microsoft.Build.Exceptions;
-using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.IO;
+using System.Linq;
+using Microsoft.Build.Construction;
+using Microsoft.Build.Exceptions;
 using Semmle.Util.Logging;
 
 namespace Semmle.Autobuild.Shared

csharp/extractor/Semmle.Extraction.CIL/Semmle.Extraction.CIL.csproj

@@ -24,7 +24,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.DiaSymReader" Version="1.4.0" />
+    <PackageReference Include="Microsoft.DiaSymReader" Version="2.0.0" />
     <PackageReference Include="Microsoft.DiaSymReader.Native" Version="1.7.0" />
     <PackageReference Include="Microsoft.DiaSymReader.PortablePdb" Version="1.6.0"><IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
 <PrivateAssets>all</PrivateAssets>

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -59,13 +59,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             this.progressMonitor.FindingFiles(srcDir);
 
             packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
-
+            var allFiles = GetAllFiles().ToList();
-            this.fileContent = new FileContent(progressMonitor, () => GetFiles("*.*"));
+            var smallFiles = allFiles.SelectSmallFiles(progressMonitor).SelectFileNames();
-            this.allSources = GetFiles("*.cs").ToList();
+            this.fileContent = new FileContent(progressMonitor, smallFiles);
-            var allProjects = GetFiles("*.csproj");
+            this.allSources = allFiles.SelectFileNamesByExtension(".cs").ToList();
+            var allProjects = allFiles.SelectFileNamesByExtension(".csproj");
             var solutions = options.SolutionFile is not null
                 ? new[] { options.SolutionFile }
-                : GetFiles("*.sln");
+                : allFiles.SelectFileNamesByExtension(".sln");
 
             var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
 
@@ -107,7 +108,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 {
                     Restore(solutions);
                     Restore(allProjects);
-                    DownloadMissingPackages();
+                    DownloadMissingPackages(allFiles);
                 }
             }
 
@@ -136,7 +137,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
                 shouldExtractWebViews)
             {
-                GenerateSourceFilesFromWebViews();
+                GenerateSourceFilesFromWebViews(allFiles);
             }
 
             progressMonitor.Summary(
@@ -151,13 +152,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 DateTime.Now - startTime);
         }
 
-        private void GenerateSourceFilesFromWebViews()
+        private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles)
         {
             progressMonitor.LogInfo($"Generating source files from cshtml and razor files.");
 
-            var views = GetFiles("*.cshtml")
+            var views = allFiles.SelectFileNamesByExtension(".cshtml", ".razor").ToArray();
-                .Concat(GetFiles("*.razor"))
-                .ToArray();
 
             if (views.Length > 0)
             {
@@ -185,15 +184,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info)) { }
 
-        private IEnumerable<string> GetFiles(string pattern, bool recurseSubdirectories = true) =>
+        private IEnumerable<FileInfo> GetAllFiles() =>
-             sourceDir.GetFiles(pattern, new EnumerationOptions
+             sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true })
-             {
+                .Where(d => d.Extension != ".dll" && !options.ExcludesFile(d.FullName));
-                 RecurseSubdirectories = recurseSubdirectories,
-                 MatchCasing = MatchCasing.CaseInsensitive
-             })
-                .Where(d => d.Extension != ".dll")
-                .Select(d => d.FullName)
-                .Where(d => !options.ExcludesFile(d));
 
         /// <summary>
         /// Computes a unique temp directory for the packages associated
@@ -374,14 +367,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
-        private void DownloadMissingPackages()
+        private void DownloadMissingPackages(List<FileInfo> allFiles)
         {
-            var nugetConfigs = GetFiles("nuget.config", recurseSubdirectories: true).ToArray();
+            var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
             string? nugetConfig = null;
             if (nugetConfigs.Length > 1)
             {
                 progressMonitor.MultipleNugetConfig(nugetConfigs);
-                nugetConfig = GetFiles("nuget.config", recurseSubdirectories: false).FirstOrDefault();
+                nugetConfig = allFiles
+                    .SelectRootFiles(sourceDir)
+                    .SelectFileNamesByName("nuget.config")
+                    .FirstOrDefault();
                 if (nugetConfig == null)
                 {
                     progressMonitor.NoTopLevelNugetConfig();
@@ -393,8 +389,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
 
             var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
-                .Select(d => Path.GetFileName(d)
+                .Select(d => Path.GetFileName(d).ToLowerInvariant());
-                .ToLowerInvariant());
             var notYetDownloadedPackages = fileContent.AllPackages.Except(alreadyDownloadedPackages);
             foreach (var package in notYetDownloadedPackages)
             {

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Text.RegularExpressions;
 using Semmle.Util;
 
@@ -18,7 +17,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     {
         private readonly ProgressMonitor progressMonitor;
         private readonly IUnsafeFileReader unsafeFileReader;
-        private readonly Func<IEnumerable<string>> getFiles;
+        private readonly IEnumerable<string> files;
         private readonly HashSet<string> allPackages = new HashSet<string>();
         private readonly Initializer initialize;
 
@@ -50,17 +49,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         }
 
         internal FileContent(ProgressMonitor progressMonitor,
-            Func<IEnumerable<string>> getFiles,
+            IEnumerable<string> files,
             IUnsafeFileReader unsafeFileReader)
         {
             this.progressMonitor = progressMonitor;
-            this.getFiles = getFiles;
+            this.files = files;
             this.unsafeFileReader = unsafeFileReader;
             this.initialize = new Initializer(DoInitialize);
         }
 
 
-        public FileContent(ProgressMonitor progressMonitor, Func<IEnumerable<string>> getFiles) : this(progressMonitor, getFiles, new UnsafeFileReader())
+        public FileContent(ProgressMonitor progressMonitor, IEnumerable<string> files) : this(progressMonitor, files, new UnsafeFileReader())
         { }
 
         private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix)
@@ -95,7 +94,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         private void DoInitialize()
         {
-            foreach (var file in getFiles())
+            foreach (var file in files)
             {
                 try
                 {

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    public static class FileInfoExtensions
+    {
+        private static IEnumerable<string> SelectFilesAux(this IEnumerable<FileInfo> files, Predicate<FileInfo> p) =>
+            files.Where(f => p(f)).Select(fi => fi.FullName);
+
+        public static IEnumerable<FileInfo> SelectRootFiles(this IEnumerable<FileInfo> files, DirectoryInfo dir) =>
+            files.Where(file => file.DirectoryName == dir.FullName);
+
+        internal static IEnumerable<FileInfo> SelectSmallFiles(this IEnumerable<FileInfo> files, ProgressMonitor progressMonitor)
+        {
+            const int oneMb = 1_048_576;
+            return files.Where(file =>
+            {
+                if (file.Length > oneMb)
+                {
+                    progressMonitor.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB.");
+                    return false;
+                }
+                return true;
+            });
+        }
+
+        public static IEnumerable<string> SelectFileNamesByExtension(this IEnumerable<FileInfo> files, params string[] extensions) =>
+            files.SelectFilesAux(fi => extensions.Contains(fi.Extension));
+
+        public static IEnumerable<string> SelectFileNamesByName(this IEnumerable<FileInfo> files, params string[] names) =>
+            files.SelectFilesAux(fi => names.Any(name => string.Compare(name, fi.Name, true) == 0));
+
+        public static IEnumerable<string> SelectFileNames(this IEnumerable<FileInfo> files) =>
+            files.SelectFilesAux(_ => true);
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
@@ -14,70 +13,91 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     /// </summary>
     internal class NugetPackages
     {
-        /// <summary>
+        private readonly string nugetExe;
-        /// Create the package manager for a specified source tree.
+        private readonly ProgressMonitor progressMonitor;
-        /// </summary>
-        public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, ProgressMonitor progressMonitor)
-        {
-            SourceDirectory = sourceDir;
-            PackageDirectory = packageDirectory;
-            this.progressMonitor = progressMonitor;
-
-            // Expect nuget.exe to be in a `nuget` directory under the directory containing this exe.
-            var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly().Location;
-            var directory = Path.GetDirectoryName(currentAssembly)
-                ?? throw new FileNotFoundException($"Directory path '{currentAssembly}' of current assembly is null");
-            nugetExe = Path.Combine(directory, "nuget", "nuget.exe");
-
-            if (!File.Exists(nugetExe))
-                throw new FileNotFoundException(string.Format("NuGet could not be found at {0}", nugetExe));
-
-            packages = new DirectoryInfo(SourceDirectory)
-                .EnumerateFiles("packages.config", SearchOption.AllDirectories)
-                .ToArray();
-        }
-
-        // List of package files to download.
-        private readonly FileInfo[] packages;
 
         /// <summary>
         /// The list of package files.
         /// </summary>
-        public IEnumerable<FileInfo> PackageFiles => packages;
+        private readonly FileInfo[] packageFiles;
-
-        /// <summary>
-        /// Download the packages to the temp folder.
-        /// </summary>
-        /// <param name="pm">The progress monitor used for reporting errors etc.</param>
-        public void InstallPackages()
-        {
-            foreach (var package in packages)
-            {
-                RestoreNugetPackage(package.FullName);
-            }
-        }
-
-        /// <summary>
-        /// The source directory used.
-        /// </summary>
-        public string SourceDirectory
-        {
-            get;
-            private set;
-        }
 
         /// <summary>
         /// The computed packages directory.
         /// This will be in the Temp location
         /// so as to not trample the source tree.
         /// </summary>
-        public TemporaryDirectory PackageDirectory { get; }
+        private readonly TemporaryDirectory packageDirectory;
+
+        /// <summary>
+        /// Create the package manager for a specified source tree.
+        /// </summary>
+        public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, ProgressMonitor progressMonitor)
+        {
+            this.packageDirectory = packageDirectory;
+            this.progressMonitor = progressMonitor;
+
+            nugetExe = ResolveNugetExe(sourceDir);
+            packageFiles = new DirectoryInfo(sourceDir)
+                .EnumerateFiles("packages.config", SearchOption.AllDirectories)
+                .ToArray();
+        }
+
+        /// <summary>
+        /// Tries to find the location of `nuget.exe` in the nuget directory under the directory
+        /// containing the executing assembly. If it can't be found, it is downloaded to the
+        /// `.nuget` directory under the source directory.
+        /// </summary>
+        /// <param name="sourceDir">The source directory.</param>
+        private string ResolveNugetExe(string sourceDir)
+        {
+            var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly().Location;
+            var directory = Path.GetDirectoryName(currentAssembly)
+                ?? throw new FileNotFoundException($"Directory path '{currentAssembly}' of current assembly is null");
+
+            var nuget = Path.Combine(directory, "nuget", "nuget.exe");
+            if (File.Exists(nuget))
+            {
+                progressMonitor.FoundNuGet(nuget);
+                return nuget;
+            }
+            else
+            {
+                progressMonitor.LogInfo($"Nuget.exe could not be found at {nuget}");
+                return DownloadNugetExe(sourceDir);
+            }
+        }
+
+        private string DownloadNugetExe(string sourceDir)
+        {
+            var directory = Path.Combine(sourceDir, ".nuget");
+            var nuget = Path.Combine(directory, "nuget.exe");
+
+            // Nuget.exe already exists in the .nuget directory.
+            if (File.Exists(nuget))
+            {
+                progressMonitor.FoundNuGet(nuget);
+                return nuget;
+            }
+
+            Directory.CreateDirectory(directory);
+            progressMonitor.LogInfo("Attempting to download nuget.exe");
+            try
+            {
+                FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget);
+                progressMonitor.LogInfo($"Downloaded nuget.exe to {nuget}");
+                return nuget;
+            }
+            catch
+            {
+                // Download failed.
+                throw new FileNotFoundException("Download of nuget.exe failed.");
+            }
+        }
 
         /// <summary>
         /// Restore all files in a specified package.
         /// </summary>
         /// <param name="package">The package file.</param>
-        /// <param name="pm">Where to log progress/errors.</param>
         private void RestoreNugetPackage(string package)
         {
             progressMonitor.NugetInstall(package);
@@ -92,12 +112,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             if (Util.Win32.IsWindows())
             {
                 exe = nugetExe;
-                args = string.Format("install -OutputDirectory {0} {1}", PackageDirectory, package);
+                args = string.Format("install -OutputDirectory {0} {1}", packageDirectory, package);
             }
             else
             {
                 exe = "mono";
-                args = string.Format("{0} install -OutputDirectory {1} {2}", nugetExe, PackageDirectory, package);
+                args = string.Format("{0} install -OutputDirectory {1} {2}", nugetExe, packageDirectory, package);
             }
 
             var pi = new ProcessStartInfo(exe, args)
@@ -133,7 +153,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
-        private readonly string nugetExe;
+        /// <summary>
-        private readonly ProgressMonitor progressMonitor;
+        /// Download the packages to the temp folder.
+        /// </summary>
+        public void InstallPackages()
+        {
+            foreach (var package in packageFiles)
+            {
+                RestoreNugetPackage(package.FullName);
+            }
+        }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs

@@ -18,7 +18,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         public void LogInfo(string message) =>
             logger.Log(Severity.Info, message);
 
-        private void LogDebug(string message) =>
+        public void LogDebug(string message) =>
             logger.Log(Severity.Debug, message);
 
         private void LogError(string message) =>
@@ -88,6 +88,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         public void MissingNuGet() =>
             LogError("Missing nuget.exe");
 
+        public void FoundNuGet(string path) =>
+            LogInfo($"Found nuget.exe at {path}");
+
         public void MissingDotNet() =>
             LogError("Missing dotnet CLI");
 

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj

@@ -19,7 +19,7 @@
 		<Folder Include="Properties\" />
 	</ItemGroup>
 	<ItemGroup>
-		<PackageReference Include="Microsoft.Build" Version="17.3.2" />
+		<PackageReference Include="Microsoft.Build" Version="17.7.2" />
 		<PackageReference Include="Microsoft.Win32.Primitives" Version="4.3.0" />
 		<PackageReference Include="System.Net.Primitives" Version="4.3.1" />
 		<PackageReference Include="System.Security.Principal" Version="4.3.0" />

csharp/extractor/Semmle.Extraction.CSharp/Semmle.Extraction.CSharp.csproj

@@ -18,7 +18,7 @@
 		<Folder Include="Properties\" />
 	</ItemGroup>
 	<ItemGroup>
-		<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.4.0" />
+		<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.7.0" />
-		<PackageReference Include="Microsoft.Build" Version="17.3.2" />
+		<PackageReference Include="Microsoft.Build" Version="17.7.2" />
 	</ItemGroup>
 </Project>

csharp/extractor/Semmle.Extraction.Tests/FileContent.cs

@@ -35,7 +35,7 @@ namespace Semmle.Extraction.Tests
     internal class TestFileContent : FileContent
     {
         public TestFileContent(List<string> lines) : base(new ProgressMonitor(new LoggerStub()),
-            () => new List<string>() { "test1.cs" },
+            new List<string>() { "test1.cs" },
             new UnsafeFileReaderStub(lines))
         { }
     }

csharp/extractor/Semmle.Extraction.Tests/Semmle.Extraction.Tests.csproj

@@ -8,12 +8,12 @@
 	<ItemGroup>
 		<PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
 		<PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
-		<PackageReference Include="xunit" Version="2.4.2" />
+		<PackageReference Include="xunit" Version="2.5.0" />
-		<PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+		<PackageReference Include="xunit.runner.visualstudio" Version="2.5.0">
 			<PrivateAssets>all</PrivateAssets>
 			<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
 		</PackageReference>
-		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
+		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.1" />
 	</ItemGroup>
 	<ItemGroup>
 		<ProjectReference Include="..\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj" />

csharp/extractor/Semmle.Extraction/Semmle.Extraction.csproj

@@ -12,9 +12,9 @@
 		<DefineConstants>TRACE;DEBUG;DEBUG_LABELS</DefineConstants>
 	</PropertyGroup>
 	<ItemGroup>
-		<PackageReference Include="Microsoft.Build" Version="17.3.2" />
+		<PackageReference Include="Microsoft.Build" Version="17.7.2" />
-		<PackageReference Include="Microsoft.CodeAnalysis" Version="4.4.0" />
+		<PackageReference Include="Microsoft.CodeAnalysis" Version="4.7.0" />
-		<PackageReference Include="GitInfo" Version="2.2.0">
+		<PackageReference Include="GitInfo" Version="3.3.1">
 			<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
 			<PrivateAssets>all</PrivateAssets>
 		</PackageReference>

csharp/extractor/Semmle.Util.Tests/Semmle.Util.Tests.csproj

@@ -6,12 +6,12 @@
 		<Nullable>enable</Nullable>
 	</PropertyGroup>
 	<ItemGroup>
-		<PackageReference Include="xunit" Version="2.4.2" />
+		<PackageReference Include="xunit" Version="2.5.0" />
-		<PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+		<PackageReference Include="xunit.runner.visualstudio" Version="2.5.0">
 			<PrivateAssets>all</PrivateAssets>
 			<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
 		</PackageReference>
-		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
+		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.1" />
 	</ItemGroup>
 	<ItemGroup>
 		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />

csharp/extractor/Semmle.Util/FileUtils.cs

@@ -1,13 +1,17 @@
 using System;
 using System.IO;
 using System.Linq;
+using System.Net.Http;
 using System.Security.Cryptography;
 using System.Text;
+using System.Threading.Tasks;
 
 namespace Semmle.Util
 {
     public static class FileUtils
     {
+        public const string NugetExeUrl = "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe";
+
         public static string ConvertToWindows(string path)
         {
             return path.Replace('/', '\\');
@@ -91,5 +95,20 @@ namespace Semmle.Util
                 hex.AppendFormat("{0:x2}", b);
             return hex.ToString();
         }
+
+        private static async Task DownloadFileAsync(string address, string filename)
+        {
+            using var httpClient = new HttpClient();
+            using var request = new HttpRequestMessage(HttpMethod.Get, address);
+            using var contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync();
+            using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
+            await contentStream.CopyToAsync(stream);
+        }
+
+        /// <summary>
+        /// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
+        /// </summary>
+        public static void DownloadFile(string address, string fileName) =>
+           DownloadFileAsync(address, fileName).Wait();
     }
 }

csharp/extractor/Semmle.Util/Semmle.Util.csproj

@@ -15,7 +15,7 @@
 
   <ItemGroup>
     <PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
   </ItemGroup>
 
 </Project>

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.6.3
+version: 1.6.4-dev
 groups:
     - csharp
     - solorigate

csharp/ql/campaigns/Solorigate/publish.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -e
 
-SOLORIGATE_ROOT="$(dirname $0)"
+SOLORIGATE_ROOT="$(readlink -f "$(dirname $0)")"
 WORKSPACE_ROOT="$SOLORIGATE_ROOT/../../../.."
 GRPS="solorigate,-test"
 

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.6.3
+version: 1.6.4-dev
 groups:
     - csharp
     - solorigate

csharp/ql/integration-tests/posix-only/dotnet_test/dotnet_test.csproj

@@ -8,10 +8,10 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.1" />
     <PackageReference Include="NUnit" Version="3.13.3" />
-    <PackageReference Include="NUnit3TestAdapter" Version="4.3.0" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.5.0" />
-    <PackageReference Include="coverlet.collector" Version="3.2.0" />
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
   </ItemGroup>
 
 </Project>

csharp/ql/integration-tests/posix-only/dotnet_test_mstest/dotnet_test_mstest.csproj

@@ -10,10 +10,10 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.10" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.1.1" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.1.1" />
-    <PackageReference Include="coverlet.collector" Version="3.1.2" />
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
   </ItemGroup>
 
 </Project>

csharp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-all
-version: 0.7.3
+version: 0.7.4-dev
 groups: csharp
 dbscheme: semmlecode.csharp.dbscheme
 extractor: csharp

csharp/ql/lib/semmle/code/csharp/Unification.qll

@@ -538,7 +538,8 @@ module Unification {
      *
      * Note: This predicate is inlined.
      */
-    bindingset[t]
+    bindingset[this]
+    pragma[inline_late]
     predicate unifiable(Type t) { none() }
 
     /**
@@ -546,7 +547,8 @@ module Unification {
      *
      * Note: This predicate is inlined.
      */
-    bindingset[t]
+    bindingset[this]
+    pragma[inline_late]
     predicate subsumes(Type t) { none() }
   }
 
@@ -554,7 +556,8 @@ module Unification {
   private class SingleConstraintTypeParameter extends ConstrainedTypeParameter {
     SingleConstraintTypeParameter() { constraintCount = 1 }
 
-    bindingset[t]
+    bindingset[this]
+    pragma[inline_late]
     override predicate unifiable(Type t) {
       exists(TTypeParameterConstraint ttc | ttc = getATypeConstraint(this) |
         ttc = TRefTypeConstraint() and
@@ -567,7 +570,8 @@ module Unification {
       )
     }
 
-    bindingset[t]
+    bindingset[this]
+    pragma[inline_late]
     override predicate subsumes(Type t) {
       exists(TTypeParameterConstraint ttc | ttc = getATypeConstraint(this) |
         ttc = TRefTypeConstraint() and
@@ -585,9 +589,13 @@ module Unification {
   private class MultiConstraintTypeParameter extends ConstrainedTypeParameter {
     MultiConstraintTypeParameter() { constraintCount > 1 }
 
-    bindingset[t]
+    pragma[nomagic]
+    TTypeParameterConstraint getATypeConstraint() { result = getATypeConstraint(this) }
+
+    bindingset[this]
+    pragma[inline_late]
     override predicate unifiable(Type t) {
-      forex(TTypeParameterConstraint ttc | ttc = getATypeConstraint(this) |
+      forex(TTypeParameterConstraint ttc | ttc = this.getATypeConstraint() |
         ttc = TRefTypeConstraint() and
         t.isRefType()
         or
@@ -598,9 +606,10 @@ module Unification {
       )
     }
 
-    bindingset[t]
+    bindingset[this]
+    pragma[inline_late]
     override predicate subsumes(Type t) {
-      forex(TTypeParameterConstraint ttc | ttc = getATypeConstraint(this) |
+      forex(TTypeParameterConstraint ttc | ttc = this.getATypeConstraint() |
         ttc = TRefTypeConstraint() and
         t.isRefType()
         or

csharp/ql/lib/semmle/code/csharp/commons/Assertions.qll

@@ -299,6 +299,12 @@ class NUnitAssertNonNullMethod extends NullnessAssertMethod, NUnitAssertMethod {
   override int getAnAssertionIndex(boolean b) { result = this.getAnAssertionIndex() and b = false }
 }
 
+pragma[nomagic]
+private predicate parameterAssertion(Assertion a, int index, Parameter p) {
+  strictcount(AssignableDefinition def | def.getTarget() = p) = 1 and
+  a.getExpr(index) = p.getAnAccess()
+}
+
 /** A method that forwards to another assertion method. */
 class ForwarderAssertMethod extends AssertMethod {
   private Assertion a;
@@ -307,10 +313,9 @@ class ForwarderAssertMethod extends AssertMethod {
 
   ForwarderAssertMethod() {
     p = this.getAParameter() and
-    strictcount(AssignableDefinition def | def.getTarget() = p) = 1 and
     forex(ControlFlowElement body | body = this.getBody() |
       bodyAsserts(this, body, a) and
-      a.getExpr(forwarderIndex) = p.getAnAccess()
+      parameterAssertion(a, forwarderIndex, p)
     )
   }
 

csharp/ql/lib/semmle/code/csharp/dataflow/internal/SsaImpl.qll

@@ -1159,6 +1159,7 @@ private predicate adjacentDefReachesUncertainRead(
   )
 }
 
+pragma[nomagic]
 private predicate adjacentDefReachesUncertainReadExt(
   DefinitionExt def, SsaInput::BasicBlock bb1, int i1, SsaInput::BasicBlock bb2, int i2
 ) {

csharp/ql/lib/semmle/code/csharp/dispatch/Dispatch.qll

@@ -862,9 +862,7 @@ private module Internal {
         or
         Unification::subsumes(t, qualifierType)
         or
-        t.(Unification::ConstrainedTypeParameter).unifiable(qualifierType)
+        qualifierType = t.(TypeParameter).getAnUltimatelySuppliedType()
-        or
-        qualifierType = t.(Unification::UnconstrainedTypeParameter).getAnUltimatelySuppliedType()
       )
     }
 

csharp/ql/src/Concurrency/UnsynchronizedStaticAccess.qhelp

@@ -12,7 +12,7 @@ invariants.
 
 <p>
 For example, the behavior of <code>Dictionary</code> when a write happens concurrently with another write or a read is 
-undefined, and frequently leads to data corruption and can lead to ​ issues as serious as livelock.
+undefined, and frequently leads to data corruption and can lead to issues as serious as livelock.
 </p>
 
 </overview>
@@ -35,6 +35,6 @@ dictionary. This means that multiple threads can access the dictionary, potentia
 
 </example>
 <references>
-  <li>MSDN, C# Reference: <a href="https://msdn.microsoft.com/en-us/library/xfhwa508.aspx#Anchor_10">Dictionary: Thread safety</a>.</li>
+  <li>MSDN, C# Reference: <a href="https://learn.microsoft.com/en-us/dotnet/api/system.collections.generic.dictionary-2?#thread-safety">Dictionary: Thread safety</a>.</li>
 </references>
 </qhelp>

csharp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-queries
-version: 0.7.3
+version: 0.7.4-dev
 groups: 
   - csharp
   - queries

csharp/ql/test/TestUtilities/InlineExpectationsTest.qll

@@ -3,19 +3,6 @@
  * See `shared/util/codeql/util/test/InlineExpectationsTest.qll`
  */
 
-private import csharp as CS
 private import codeql.util.test.InlineExpectationsTest
-
+private import internal.InlineExpectationsTestImpl
-private module Impl implements InlineExpectationsTestSig {
-  /**
-   * A class representing line comments in C# used by the InlineExpectations core code
-   */
-  class ExpectationComment extends CS::SinglelineComment {
-    /** Gets the contents of the given comment, _without_ the preceding comment marker (`//`). */
-    string getContents() { result = this.getText() }
-  }
-
-  class Location = CS::Location;
-}
-
 import Make<Impl>

csharp/ql/test/TestUtilities/InlineFlowTest.qll

@@ -1,120 +1,34 @@
 /**
- * Provides a simple base test for flow-related tests using inline expectations.
+ * Inline flow tests for CSharp.
- *
+ * See `shared/util/codeql/dataflow/test/InlineFlowTest.qll`
- * Example for a test.ql:
- * ```ql
- * import csharp
- * import TestUtilities.InlineFlowTest
- * import DefaultFlowTest
- * import PathGraph
- *
- * from PathNode source, PathNode sink
- * where flowPath(source, sink)
- * select sink, source, sink, "$@", source, source.toString()
- *
- * ```
- *
- * To declare expectations, you can use the $hasTaintFlow or $hasValueFlow comments within the test source files.
- * Example of the corresponding test file, e.g. Test.cs
- * ```csharp
- * public class Test
- * {
- *     object Source() { return null; }
- *     string Taint() { return null; }
- *     void Sink(object o) { }
- *
- *     public void test()
- *     {
- *         var s = Source(1);
- *         Sink(s); // $ hasValueFlow=1
- *         var t = "foo" + Taint(2);
- *         Sink(t); // $ hasTaintFlow=2
- *     }
- * }
- * ```
- *
- * If you are only interested in value flow, then instead of importing `DefaultFlowTest`, you can import
- * `ValueFlowTest<DefaultFlowConfig>`. Similarly, if you are only interested in taint flow, then instead of
- * importing `DefaultFlowTest`, you can import `TaintFlowTest<DefaultFlowConfig>`. In both cases
- * `DefaultFlowConfig` can be replaced by another implementation of `DataFlow::ConfigSig`.
- *
- * If you need more fine-grained tuning, consider implementing a test using `InlineExpectationsTest`.
  */
 
 import csharp
-import TestUtilities.InlineExpectationsTest
+private import codeql.dataflow.test.InlineFlowTest
+private import semmle.code.csharp.dataflow.internal.DataFlowImplSpecific
+private import semmle.code.csharp.dataflow.internal.TaintTrackingImplSpecific
+private import internal.InlineExpectationsTestImpl
 
-private predicate defaultSource(DataFlow::Node source) {
+private module FlowTestImpl implements InputSig<CsharpDataFlow> {
+  predicate defaultSource(DataFlow::Node source) {
     source.asExpr().(MethodCall).getTarget().getUndecoratedName() = ["Source", "Taint"]
   }
 
-private predicate defaultSink(DataFlow::Node sink) {
+  predicate defaultSink(DataFlow::Node sink) {
     exists(MethodCall mc | mc.getTarget().hasUndecoratedName("Sink") |
       sink.asExpr() = mc.getAnArgument()
     )
   }
 
-module DefaultFlowConfig implements DataFlow::ConfigSig {
-  predicate isSource(DataFlow::Node source) { defaultSource(source) }
-
-  predicate isSink(DataFlow::Node sink) { defaultSink(sink) }
-
-  int fieldFlowBranchLimit() { result = 1000 }
-}
-
-private module NoFlowConfig implements DataFlow::ConfigSig {
-  predicate isSource(DataFlow::Node source) { none() }
-
-  predicate isSink(DataFlow::Node sink) { none() }
-}
-
   private string getSourceArgString(DataFlow::Node src) {
     defaultSource(src) and
     src.asExpr().(MethodCall).getAnArgument().getValue() = result
   }
 
-module FlowTest<DataFlow::ConfigSig ValueFlowConfig, DataFlow::ConfigSig TaintFlowConfig> {
+  string getArgString(DataFlow::Node src, DataFlow::Node sink) {
-  module ValueFlow = DataFlow::Global<ValueFlowConfig>;
+    (if exists(getSourceArgString(src)) then result = getSourceArgString(src) else result = "") and
-
+    exists(sink)
-  module TaintFlow = TaintTracking::Global<TaintFlowConfig>;
-
-  private module InlineTest implements TestSig {
-    string getARelevantTag() { result = ["hasValueFlow", "hasTaintFlow"] }
-
-    predicate hasActualResult(Location location, string element, string tag, string value) {
-      tag = "hasValueFlow" and
-      exists(DataFlow::Node src, DataFlow::Node sink | ValueFlow::flow(src, sink) |
-        sink.getLocation() = location and
-        element = sink.toString() and
-        if exists(getSourceArgString(src)) then value = getSourceArgString(src) else value = ""
-      )
-      or
-      tag = "hasTaintFlow" and
-      exists(DataFlow::Node src, DataFlow::Node sink |
-        TaintFlow::flow(src, sink) and not ValueFlow::flow(src, sink)
-      |
-        sink.getLocation() = location and
-        element = sink.toString() and
-        if exists(getSourceArgString(src)) then value = getSourceArgString(src) else value = ""
-      )
   }
 }
 
-  import MakeTest<InlineTest>
+import InlineFlowTestMake<CsharpDataFlow, CsharpTaintTracking, Impl, FlowTestImpl>
-  import DataFlow::MergePathGraph<ValueFlow::PathNode, TaintFlow::PathNode, ValueFlow::PathGraph, TaintFlow::PathGraph>
-
-  predicate flowPath(PathNode source, PathNode sink) {
-    ValueFlow::flowPath(source.asPathNode1(), sink.asPathNode1()) or
-    TaintFlow::flowPath(source.asPathNode2(), sink.asPathNode2())
-  }
-}
-
-module DefaultFlowTest = FlowTest<DefaultFlowConfig, DefaultFlowConfig>;
-
-module ValueFlowTest<DataFlow::ConfigSig ValueFlowConfig> {
-  import FlowTest<ValueFlowConfig, NoFlowConfig>
-}
-
-module TaintFlowTest<DataFlow::ConfigSig TaintFlowConfig> {
-  import FlowTest<NoFlowConfig, TaintFlowConfig>
-}

csharp/ql/test/TestUtilities/internal/InlineExpectationsTestImpl.qll

@@ -0,0 +1,14 @@
+private import csharp as CS
+private import codeql.util.test.InlineExpectationsTest
+
+module Impl implements InlineExpectationsTestSig {
+  /**
+   * A class representing line comments in C# used by the InlineExpectations core code
+   */
+  class ExpectationComment extends CS::SinglelineComment {
+    /** Gets the contents of the given comment, _without_ the preceding comment marker (`//`). */
+    string getContents() { result = this.getText() }
+  }
+
+  class Location = CS::Location;
+}

csharp/ql/test/library-tests/dataflow/fields/FieldFlow.expected

@@ -1,4 +1,3 @@
-failures
 testFailures
 edges
 | A.cs:5:17:5:28 | call to method Source<C> : C | A.cs:6:24:6:24 | access to local variable c : C |

csharp/ql/test/library-tests/dataflow/operators/operatorFlow.expected

@@ -1,4 +1,3 @@
-failures
 testFailures
 edges
 | Operator.cs:9:39:9:39 | x : C | Operator.cs:9:50:9:50 | access to parameter x : C |

csharp/ql/test/library-tests/dataflow/patterns/PatternFlow.expected

@@ -1,4 +1,3 @@
-failures
 testFailures
 edges
 nodes

csharp/ql/test/library-tests/dataflow/tuples/Tuples.expected

@@ -1,4 +1,3 @@
-failures
 testFailures
 edges
 | Tuples.cs:7:18:7:34 | call to method Source<Object> : Object | Tuples.cs:10:21:10:22 | access to local variable o1 : Object |

csharp/ql/test/query-tests/Dead Code/NonAssignedFields/options

@@ -0,0 +1,2 @@
+semmle-extractor-options: /nostdlib /noconfig
+semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/_frameworks/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj

csharp/ql/test/query-tests/Dead Code/Tests/options

@@ -1 +1,3 @@
-semmle-extractor-options: /r:System.Dynamic.Runtime.dll /r:System.Linq.Expressions.dll ${testdir}/../../../resources/stubs/Microsoft.VisualStudio.TestTools.UnitTesting.cs
+semmle-extractor-options: /nostdlib /noconfig
+semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/_frameworks/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj
+semmle-extractor-options: ${testdir}/../../../resources/stubs/Microsoft.VisualStudio.TestTools.UnitTesting.cs

csharp/ql/test/query-tests/Documentation/options

@@ -0,0 +1,2 @@
+semmle-extractor-options: /nostdlib /noconfig
+semmle-extractor-options: --load-sources-from-project:${testdir}/../../resources/stubs/_frameworks/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj

csharp/ql/test/query-tests/EmptyBlock/options

@@ -0,0 +1,2 @@
+semmle-extractor-options: /nostdlib /noconfig
+semmle-extractor-options: --load-sources-from-project:${testdir}/../../resources/stubs/_frameworks/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj

csharp/ql/test/query-tests/Language Abuse/CatchOfGenericException/options

@@ -0,0 +1,2 @@
+semmle-extractor-options: /nostdlib /noconfig
+semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/_frameworks/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj

csharp/ql/test/query-tests/Language Abuse/ChainedIs/options

@@ -0,0 +1,2 @@
+semmle-extractor-options: /nostdlib /noconfig
+semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/_frameworks/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj

csharp/ql/test/query-tests/Language Abuse/DubiousTypeTestOfThis/options

@@ -1 +1,2 @@
-semmle-extractor-options: /r:System.Diagnostics.Debug.dll
+semmle-extractor-options: /nostdlib /noconfig
+semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/_frameworks/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj

csharp/ql/test/query-tests/Language Abuse/ForeachCapture/options

@@ -1 +1,2 @@
-semmle-extractor-options: /r:System.Runtime.Extensions.dll
+semmle-extractor-options: /nostdlib /noconfig
+semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/_frameworks/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj

csharp/ql/test/query-tests/Language Abuse/MissedReadonlyOpportunity/options

@@ -0,0 +1,2 @@
+semmle-extractor-options: /nostdlib /noconfig
+semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/_frameworks/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj

csharp/ql/test/query-tests/Language Abuse/MissedTernaryOpportunity/options

@@ -0,0 +1,2 @@
+semmle-extractor-options: /nostdlib /noconfig
+semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/_frameworks/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj

csharp/ql/test/query-tests/Language Abuse/NestedIf/options

@@ -0,0 +1,2 @@
+semmle-extractor-options: /nostdlib /noconfig
+semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/_frameworks/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj

csharp/ql/test/query-tests/Language Abuse/RethrowException/options

@@ -0,0 +1,2 @@
+semmle-extractor-options: /nostdlib /noconfig
+semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/_frameworks/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj

csharp/ql/test/query-tests/Language Abuse/SimplifyBoolExpr/options

@@ -0,0 +1,2 @@
+semmle-extractor-options: /nostdlib /noconfig
+semmle-extractor-options: --load-sources-from-project:${testdir}/../../../resources/stubs/_frameworks/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj