Merge pull request #16840 from yoff/stdlib-optparse

Python: Modelling of the Standard Library

.bazelversion

@@ -1 +1 @@
-7.1.2
+7.2.0

.github/workflows/ruby-build.yml

@@ -7,6 +7,7 @@ on:
       - .github/workflows/ruby-build.yml
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
+      - "shared/tree-sitter-extractor/**"
     branches:
       - main
       - "rc/*"
@@ -16,6 +17,7 @@ on:
       - .github/workflows/ruby-build.yml
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
+      - "shared/tree-sitter-extractor/**"
     branches:
       - main
       - "rc/*"

MODULE.bazel

@@ -13,22 +13,45 @@ local_path_override(
 
 # see https://registry.bazel.build/ for a list of available packages
 
-bazel_dep(name = "platforms", version = "0.0.9")
+bazel_dep(name = "platforms", version = "0.0.10")
-bazel_dep(name = "rules_go", version = "0.47.0")
+bazel_dep(name = "rules_go", version = "0.48.0")
 bazel_dep(name = "rules_pkg", version = "0.10.1")
-bazel_dep(name = "rules_nodejs", version = "6.0.3")
+bazel_dep(name = "rules_nodejs", version = "6.2.0-codeql.1")
-bazel_dep(name = "rules_python", version = "0.31.0")
+bazel_dep(name = "rules_python", version = "0.32.2")
-bazel_dep(name = "bazel_skylib", version = "1.5.0")
+bazel_dep(name = "bazel_skylib", version = "1.6.1")
 bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl")
 bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
 bazel_dep(name = "fmt", version = "10.0.0")
 bazel_dep(name = "rules_kotlin", version = "1.9.4-codeql.1")
-bazel_dep(name = "gazelle", version = "0.36.0")
+bazel_dep(name = "gazelle", version = "0.37.0")
 bazel_dep(name = "rules_dotnet", version = "0.15.1")
 bazel_dep(name = "googletest", version = "1.14.0.bcr.1")
+bazel_dep(name = "rules_rust", version = "0.46.0")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
+crate = use_extension(
+    "@rules_rust//crate_universe:extension.bzl",
+    "crate",
+)
+crate.from_cargo(
+    name = "py_deps",
+    cargo_lockfile = "//python/extractor/tsg-python:Cargo.lock",
+    manifests = [
+        "//python/extractor/tsg-python:Cargo.toml",
+        "//python/extractor/tsg-python/tsp:Cargo.toml",
+    ],
+)
+crate.from_cargo(
+    name = "ruby_deps",
+    cargo_lockfile = "//ruby/extractor:Cargo.lock",
+    manifests = [
+        "//ruby/extractor:Cargo.toml",
+        "//ruby/extractor/codeql-extractor-fake-crate:Cargo.toml",
+    ],
+)
+use_repo(crate, "py_deps", "ruby_deps")
+
 dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet")
 dotnet.toolchain(dotnet_version = "8.0.101")
 use_repo(dotnet, "dotnet_toolchains")
@@ -62,6 +85,10 @@ use_repo(
 node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node")
 node.toolchain(
     name = "nodejs",
+    node_urls = [
+        "https://nodejs.org/dist/v{version}/{filename}",
+        "https://mirrors.dotsrc.org/nodejs/release/v{version}/{filename}",
+    ],
     node_version = "18.15.0",
 )
 use_repo(node, "nodejs", "nodejs_toolchains")

config/identical-files.json

@@ -61,10 +61,6 @@
     "java/ql/src/utils/modelgenerator/internal/CaptureModels.qll",
     "csharp/ql/src/utils/modelgenerator/internal/CaptureModels.qll"
   ],
-  "Model as Data Generation Java/C# - CaptureModelsPrinting": [
-    "java/ql/src/utils/modelgenerator/internal/CaptureModelsPrinting.qll",
-    "csharp/ql/src/utils/modelgenerator/internal/CaptureModelsPrinting.qll"
-  ],
   "Sign Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/Sign.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/Sign.qll"
@@ -185,11 +181,6 @@
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/AliasAnalysisImports.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/AliasAnalysisImports.qll"
   ],
-  "C++ IR ValueNumberingImports": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/gvn/internal/ValueNumberingImports.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/gvn/internal/ValueNumberingImports.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/gvn/internal/ValueNumberingImports.qll"
-  ],
   "IR SSA SSAConstruction": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/SSAConstruction.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/SSAConstruction.qll"

cpp/ql/lib/change-notes/2024-06-10-builtin-expect.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The "Guards" library (`semmle.code.cpp.controlflow.Guards`) now also infers guards from calls to the builtin operation `__builtin_expect`. As a result, some queries may produce fewer false positives.

cpp/ql/lib/change-notes/2024-06-13-double-free.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The queries "Potential double free" (`cpp/double-free`) and "Potential use after free" (`cpp/use-after-free`) now produce fewer false positives.

cpp/ql/lib/change-notes/2024-06-20-extensible-allocation-deallocation.md

@@ -0,0 +1,4 @@
+---
+category: feature
+---
+* It is now possible to extend the classes `AllocationFunction` and `DeallocationFunction` via data extensions. Extensions of these classes should be added to the `lib/ext/allocation` and `lib/ext/deallocation` directories respectively.

cpp/ql/lib/ext/allocation/Bsd.allocation.model.yml

@@ -0,0 +1,7 @@
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: allocationFunctionModel
+    data:
+      - ["", "", False, "kmem_alloc", "0", "", "", True]
+      - ["", "", False, "kmem_zalloc", "0", "", "", True]

cpp/ql/lib/ext/allocation/Glibc.allocation.model.yml

@@ -0,0 +1,7 @@
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: allocationFunctionModel
+    data:
+      - ["", "", False, "g_malloc", "0", "", "", True]
+      - ["", "", False, "g_try_malloc", "0", "", "", True]

cpp/ql/lib/ext/allocation/OpenSSL.allocation.model.yml

@@ -0,0 +1,10 @@
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: allocationFunctionModel
+    data:
+      - ["", "", False, "CRYPTO_malloc", "0", "", "", True]
+      - ["", "", False, "CRYPTO_zalloc", "0", "", "", True]
+      - ["", "", False, "CRYPTO_secure_malloc", "0", "", "", True]
+      - ["", "", False, "CRYPTO_secure_zalloc", "0", "", "", True]
+

cpp/ql/lib/ext/allocation/Std.allocation.model.yml

@@ -0,0 +1,15 @@
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: allocationFunctionModel
+    data:
+      - ["", "", False, "malloc", "0", "", "", True]
+      - ["std", "", False, "malloc", "0", "", "", True]
+      - ["bsl", "", False, "malloc", "0", "", "", True]
+      - ["", "", False, "alloca", "0", "", "", False]
+      - ["", "", False, "__builtin_alloca", "0", "", "", False]
+      - ["", "", False, "_alloca", "0", "", "", False]
+      - ["", "", False, "_malloca", "0", "", "", False]
+      - ["", "", False, "calloc", "1", "0", "", True]
+      - ["std", "", False, "calloc", "1", "0", "", True]
+      - ["bsl", "", False, "calloc", "1", "0", "", True]

cpp/ql/lib/ext/allocation/Windows.allocation.model.yml

@@ -0,0 +1,29 @@
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: allocationFunctionModel
+    data:
+      - ["", "", False, "MmAllocateContiguousMemory", "0", "", "", True]
+      - ["", "", False, "MmAllocateContiguousNodeMemory", "0", "", "", True]
+      - ["", "", False, "MmAllocateContiguousMemorySpecifyCache", "0", "", "", True]
+      - ["", "", False, "MmAllocateContiguousMemorySpecifyCacheNode", "0", "", "", True]
+      - ["", "", False, "MmAllocateNonCachedMemory", "0", "", "", True]
+      - ["", "", False, "MmAllocateMappingAddress", "0", "", "", True]
+      - ["", "", False, "CoTaskMemAlloc", "0", "", "", True]
+      - ["", "", False, "ExAllocatePool", "1", "", "", True]
+      - ["", "", False, "ExAllocatePool2", "1", "", "", True]
+      - ["", "", False, "ExAllocatePool3", "1", "", "", True]
+      - ["", "", False, "ExAllocatePoolWithTag", "1", "", "", True]
+      - ["", "", False, "ExAllocatePoolWithTagPriority", "1", "", "", True]
+      - ["", "", False, "ExAllocatePoolWithQuota", "1", "", "", True]
+      - ["", "", False, "ExAllocatePoolWithQuotaTag", "1", "", "", True]
+      - ["", "", False, "ExAllocatePoolZero", "1", "", "", True]
+      - ["", "", False, "IoAllocateMdl", "1", "", "", True]
+      - ["", "", False, "IoAllocateErrorLogEntry", "1", "", "", True]
+      - ["", "", False, "LocalAlloc", "1", "", "", True]
+      - ["", "", False, "GlobalAlloc", "1", "", "", True]
+      - ["", "", False, "VirtualAlloc", "1", "", "", True]
+      - ["", "", False, "HeapAlloc", "2", "", "", True]
+      - ["", "", False, "MmAllocatePagesForMdl", "3", "", "", True]
+      - ["", "", False, "MmAllocatePagesForMdlEx", "3", "", "", True]
+      - ["", "", False, "MmAllocateNodePagesForMdlEx", "3", "", "", True]

cpp/ql/lib/ext/allocation/empty.allocation.model.yml

@@ -0,0 +1,5 @@
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: allocationFunctionModel
+    data: []

cpp/ql/lib/ext/deallocation/Bsd.deallocation.model.yml

@@ -0,0 +1,8 @@
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: deallocationFunctionModel
+    data:
+      - ["", "", False, "pool_put", "1"]
+      - ["", "", False, "pool_cache_put", "1"]
+      - ["", "", False, "kmem_free", "0"]

cpp/ql/lib/ext/deallocation/Std.deallocation.model.yml

@@ -0,0 +1,42 @@
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: deallocationFunctionModel
+    data:
+      - ["", "", False, "free", "0"]
+      - ["std", "", False, "free", "0"]
+      - ["bsl", "", False, "free", "0"]
+      - ["", "", False, "realloc", "0"]
+      - ["std", "", False, "realloc", "0"]
+      - ["bsl", "", False, "realloc", "0"]
+      - ["", "", False, "CRYPTO_free", "0"]
+      - ["", "", False, "CRYPTO_secure_free", "0"]
+      - ["", "", False, "g_free", "0"]
+      - ["", "", False, "ExFreePool", "0"]
+      - ["", "", False, "ExFreePoolWithTag", "0"]
+      - ["", "", False, "ExDeleteTimer", "0"]
+      - ["", "", False, "IoFreeIrp", "0"]
+      - ["", "", False, "IoFreeMdl", "0"]
+      - ["", "", False, "IoFreeErrorLogEntry", "0"]
+      - ["", "", False, "IoFreeWorkItem", "0"]
+      - ["", "", False, "MmFreeContiguousMemory", "0"]
+      - ["", "", False, "MmFreeContiguousMemorySpecifyCache", "0"]
+      - ["", "", False, "MmFreeNonCachedMemory", "0"]
+      - ["", "", False, "MmFreeMappingAddress", "0"]
+      - ["", "", False, "MmFreePagesFromMdl", "0"]
+      - ["", "", False, "MmUnmapReservedMapping", "0"]
+      - ["", "", False, "MmUnmapLockedPages", "0"]
+      - ["", "", False, "NdisFreeGenericObject", "0"]
+      - ["", "", False, "NdisFreeMemory", "0"]
+      - ["", "", False, "NdisFreeMemoryWithTag", "0"]
+      - ["", "", False, "NdisFreeMdl", "0"]
+      - ["", "", False, "NdisFreeNetBufferListPool", "0"]
+      - ["", "", False, "NdisFreeNetBufferPool", "0"]
+      - ["", "", False, "LocalFree", "0"]
+      - ["", "", False, "GlobalFree", "0"]
+      - ["", "", False, "LocalReAlloc", "0"]
+      - ["", "", False, "GlobalReAlloc", "0"]
+      - ["", "", False, "VirtualFree", "0"]
+      - ["", "", False, "CoTaskMemFree", "0"]
+      - ["", "", False, "CoTaskMemRealloc", "0"]
+      - ["", "", False, "SysFreeString", "0"]

cpp/ql/lib/ext/deallocation/Windows.deallocation.model.yml

@@ -0,0 +1,41 @@
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: deallocationFunctionModel
+    data:
+      - ["", "", False, "ExFreePool", "0"]
+      - ["", "", False, "ExFreePoolWithTag", "0"]
+      - ["", "", False, "ExDeleteTimer", "0"]
+      - ["", "", False, "IoFreeIrp", "0"]
+      - ["", "", False, "IoFreeMdl", "0"]
+      - ["", "", False, "IoFreeErrorLogEntry", "0"]
+      - ["", "", False, "IoFreeWorkItem", "0"]
+      - ["", "", False, "MmFreeContiguousMemory", "0"]
+      - ["", "", False, "MmFreeContiguousMemorySpecifyCache", "0"]
+      - ["", "", False, "MmFreeNonCachedMemory", "0"]
+      - ["", "", False, "MmFreeMappingAddress", "0"]
+      - ["", "", False, "MmFreePagesFromMdl", "0"]
+      - ["", "", False, "MmUnmapReservedMapping", "0"]
+      - ["", "", False, "MmUnmapLockedPages", "0"]
+      - ["", "", False, "NdisFreeGenericObject", "0"]
+      - ["", "", False, "NdisFreeMemory", "0"]
+      - ["", "", False, "NdisFreeMemoryWithTag", "0"]
+      - ["", "", False, "NdisFreeMdl", "0"]
+      - ["", "", False, "NdisFreeNetBufferListPool", "0"]
+      - ["", "", False, "NdisFreeNetBufferPool", "0"]
+      - ["", "", False, "LocalFree", "0"]
+      - ["", "", False, "GlobalFree", "0"]
+      - ["", "", False, "LocalReAlloc", "0"]
+      - ["", "", False, "GlobalReAlloc", "0"]
+      - ["", "", False, "VirtualFree", "0"]
+      - ["", "", False, "CoTaskMemFree", "0"]
+      - ["", "", False, "CoTaskMemRealloc", "0"]
+      - ["", "", False, "SysFreeString", "0"]
+      - ["", "", False, "ExFreeToLookasideListEx", "1"]
+      - ["", "", False, "ExFreeToPagedLookasideList", "1"]
+      - ["", "", False, "ExFreeToNPagedLookasideList", "1"]
+      - ["", "", False, "NdisFreeMemoryWithTagPriority", "1"]
+      - ["", "", False, "StorPortFreeMdl", "1"]
+      - ["", "", False, "StorPortFreePool", "1"]
+      - ["", "", False, "HeapFree", "2"]
+      - ["", "", False, "HeapReAlloc", "2"]

cpp/ql/lib/ext/deallocation/empty.deallocation.model.yml

@@ -0,0 +1,5 @@
+extensions:
+  - addsTo:
+      pack: codeql/cpp-all
+      extensible: deallocationFunctionModel
+    data: []

cpp/ql/lib/qlpack.yml

@@ -16,4 +16,6 @@ dependencies:
   codeql/xml: ${workspace}
 dataExtensions:
   - ext/*.model.yml
+  - ext/deallocation/*.model.yml
+  - ext/allocation/*.model.yml
 warnOnImplicitThis: true

cpp/ql/lib/semmle/code/cpp/controlflow/IRGuards.qll

@@ -762,6 +762,8 @@ private predicate compares_eq(
   exists(AbstractValue dual | value = dual.getDualValue() |
     compares_eq(test.(LogicalNotInstruction).getUnary(), left, right, k, areEqual, dual)
   )
+  or
+  compares_eq(test.(BuiltinExpectCallInstruction).getCondition(), left, right, k, areEqual, value)
 }
 
 /**
@@ -831,6 +833,9 @@ private predicate unary_compares_eq(
     int_value(const) = k1 and
     k = k1 + k2
   )
+  or
+  unary_compares_eq(test.(BuiltinExpectCallInstruction).getCondition(), op, k, areEqual,
+    inNonZeroCase, value)
 }
 
 /** Rearrange various simple comparisons into `left == right + k` form. */
@@ -910,12 +915,68 @@ private predicate unary_simple_comparison_eq(
   )
 }
 
+/** A call to the builtin operation `__builtin_expect`. */
+private class BuiltinExpectCallInstruction extends CallInstruction {
+  BuiltinExpectCallInstruction() { this.getStaticCallTarget().hasName("__builtin_expect") }
+
+  /** Gets the condition of this call. */
+  Instruction getCondition() {
+    // The first parameter of `__builtin_expect` has type `long`. So we skip
+    // the conversion when inferring guards.
+    result = this.getArgument(0).(ConvertInstruction).getUnary()
+  }
+}
+
+/**
+ * Holds if `left == right + k` is `areEqual` if `cmp` evaluates to `value`,
+ * and `cmp` is an instruction that compares the value of
+ * `__builtin_expect(left == right + k, _)` to `0`.
+ */
+private predicate builtin_expect_eq(
+  CompareInstruction cmp, Operand left, Operand right, int k, boolean areEqual, AbstractValue value
+) {
+  exists(BuiltinExpectCallInstruction call, Instruction const, AbstractValue innerValue |
+    int_value(const) = 0 and
+    cmp.hasOperands(call.getAUse(), const.getAUse()) and
+    compares_eq(call.getCondition(), left, right, k, areEqual, innerValue)
+  |
+    cmp instanceof CompareNEInstruction and
+    value = innerValue
+    or
+    cmp instanceof CompareEQInstruction and
+    value.getDualValue() = innerValue
+  )
+}
+
 private predicate complex_eq(
   CompareInstruction cmp, Operand left, Operand right, int k, boolean areEqual, AbstractValue value
 ) {
   sub_eq(cmp, left, right, k, areEqual, value)
   or
   add_eq(cmp, left, right, k, areEqual, value)
+  or
+  builtin_expect_eq(cmp, left, right, k, areEqual, value)
+}
+
+/**
+ * Holds if `op == k` is `areEqual` if `cmp` evaluates to `value`, and `cmp` is
+ * an instruction that compares the value of `__builtin_expect(op == k, _)` to `0`.
+ */
+private predicate unary_builtin_expect_eq(
+  CompareInstruction cmp, Operand op, int k, boolean areEqual, boolean inNonZeroCase,
+  AbstractValue value
+) {
+  exists(BuiltinExpectCallInstruction call, Instruction const, AbstractValue innerValue |
+    int_value(const) = 0 and
+    cmp.hasOperands(call.getAUse(), const.getAUse()) and
+    unary_compares_eq(call.getCondition(), op, k, areEqual, inNonZeroCase, innerValue)
+  |
+    cmp instanceof CompareNEInstruction and
+    value = innerValue
+    or
+    cmp instanceof CompareEQInstruction and
+    value.getDualValue() = innerValue
+  )
 }
 
 private predicate unary_complex_eq(
@@ -924,6 +985,8 @@ private predicate unary_complex_eq(
   unary_sub_eq(test, op, k, areEqual, inNonZeroCase, value)
   or
   unary_add_eq(test, op, k, areEqual, inNonZeroCase, value)
+  or
+  unary_builtin_expect_eq(test, op, k, areEqual, inNonZeroCase, value)
 }
 
 /*

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowPrivate.qll

@@ -216,7 +216,7 @@ predicate localMustFlowStep(Node node1, Node node2) { none() }
 
 /** Gets the type of `n` used for type pruning. */
 Type getNodeType(Node n) {
-  suppressUnusedNode(n) and
+  exists(n) and
   result instanceof VoidType // stub implementation
 }
 
@@ -227,13 +227,10 @@ string ppReprType(Type t) { none() } // stub implementation
  * Holds if `t1` and `t2` are compatible, that is, whether data can flow from
  * a node of type `t1` to a node of type `t2`.
  */
-pragma[inline]
 predicate compatibleTypes(Type t1, Type t2) {
-  any() // stub implementation
+  t1 instanceof VoidType and t2 instanceof VoidType // stub implementation
 }
 
-private predicate suppressUnusedNode(Node n) { any() }
-
 //////////////////////////////////////////////////////////////////////////////
 // Java QL library compatibility wrappers
 //////////////////////////////////////////////////////////////////////////////

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll

@@ -988,7 +988,7 @@ predicate localMustFlowStep(Node node1, Node node2) { none() }
 
 /** Gets the type of `n` used for type pruning. */
 DataFlowType getNodeType(Node n) {
-  suppressUnusedNode(n) and
+  exists(n) and
   result instanceof VoidType // stub implementation
 }
 
@@ -999,13 +999,10 @@ string ppReprType(DataFlowType t) { none() } // stub implementation
  * Holds if `t1` and `t2` are compatible, that is, whether data can flow from
  * a node of type `t1` to a node of type `t2`.
  */
-pragma[inline]
 predicate compatibleTypes(DataFlowType t1, DataFlowType t2) {
-  any() // stub implementation
+  t1 instanceof VoidType and t2 instanceof VoidType // stub implementation
 }
 
-private predicate suppressUnusedNode(Node n) { any() }
-
 //////////////////////////////////////////////////////////////////////////////
 // Java QL library compatibility wrappers
 //////////////////////////////////////////////////////////////////////////////

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/ExprNodes.qll

@@ -193,6 +193,46 @@ private module Cached {
     )
   }
 
+  /** Holds if `operand`'s definition is a `VariableAddressInstruction` whose variable is a temporary */
+  private predicate isIRTempVariable(Operand operand) {
+    operand.getDef().(VariableAddressInstruction).getIRVariable() instanceof IRTempVariable
+  }
+
+  /**
+   * Holds if `node` is an indirect operand whose operand is an argument, and
+   * the `n`'th expression associated with the operand is `e`.
+   */
+  private predicate isIndirectOperandOfArgument(
+    IndirectOperand node, ArgumentOperand operand, Expr e, int n
+  ) {
+    node.hasOperandAndIndirectionIndex(operand, 1) and
+    e = getConvertedResultExpression(operand.getDef(), n)
+  }
+
+  /**
+   * Holds if `opFrom` is an operand to a conversion, and `opTo` is the unique
+   * use of the conversion.
+   */
+  private predicate isConversionStep(Operand opFrom, Operand opTo) {
+    exists(Instruction mid |
+      conversionFlow(opFrom, mid, false, false) and
+      opTo = unique( | | getAUse(mid))
+    )
+  }
+
+  /**
+   * Holds if an operand that satisfies `isIRTempVariable` flows to `op`
+   * through a (possibly empty) sequence of conversions.
+   */
+  private predicate irTempOperandConversionFlows(Operand op) {
+    isIRTempVariable(op)
+    or
+    exists(Operand mid |
+      irTempOperandConversionFlows(mid) and
+      isConversionStep(mid, op)
+    )
+  }
+
   /** Holds if `node` should be an `IndirectOperand` that maps `node.asExpr()` to `e`. */
   private predicate exprNodeShouldBeIndirectOperand(IndirectOperand node, Expr e, int n) {
     exists(ArgumentOperand operand |
@@ -203,9 +243,8 @@ private module Cached {
       // result. However, the instruction actually represents the _address_ of
       // the argument. So to fix this mismatch, we have the indirection of the
       // `VariableAddressInstruction` map to the expression.
-      node.hasOperandAndIndirectionIndex(operand, 1) and
+      isIndirectOperandOfArgument(node, operand, e, n) and
-      e = getConvertedResultExpression(operand.getDef(), n) and
+      irTempOperandConversionFlows(operand)
-      operand.getDef().(VariableAddressInstruction).getIRVariable() instanceof IRTempVariable
     )
   }
 

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/gvn/internal/ValueNumberingImports.qll

@@ -1,3 +1,3 @@
-import semmle.code.cpp.ir.implementation.aliased_ssa.IR
+import semmle.code.cpp.ir.implementation.raw.IR
 import semmle.code.cpp.ir.internal.Overlap
 import semmle.code.cpp.ir.internal.IRCppLanguage as Language

cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/gvn/internal/ValueNumberingImports.qll

@@ -1,3 +1,3 @@
-import semmle.code.cpp.ir.implementation.aliased_ssa.IR
+import semmle.code.cpp.ir.implementation.unaliased_ssa.IR
 import semmle.code.cpp.ir.internal.Overlap
 import semmle.code.cpp.ir.internal.IRCppLanguage as Language

cpp/ql/lib/semmle/code/cpp/models/implementations/Allocation.qll

@@ -7,119 +7,6 @@
 import semmle.code.cpp.models.interfaces.Allocation
 import semmle.code.cpp.models.interfaces.Taint
 
-/**
- * An allocation function (such as `malloc`) that has an argument for the size
- * in bytes.
- */
-private class MallocAllocationFunction extends AllocationFunction {
-  int sizeArg;
-
-  MallocAllocationFunction() {
-    // --- C library allocation
-    this.hasGlobalOrStdOrBslName("malloc") and // malloc(size)
-    sizeArg = 0
-    or
-    this.hasGlobalName([
-        // --- Windows Memory Management for Windows Drivers
-        "MmAllocateContiguousMemory", // MmAllocateContiguousMemory(size, maxaddress)
-        "MmAllocateContiguousNodeMemory", // MmAllocateContiguousNodeMemory(size, minaddress, maxaddress, bound, flag, prefer)
-        "MmAllocateContiguousMemorySpecifyCache", // MmAllocateContiguousMemorySpecifyCache(size, minaddress, maxaddress, bound, type)
-        "MmAllocateContiguousMemorySpecifyCacheNode", // MmAllocateContiguousMemorySpecifyCacheNode(size, minaddress, maxaddress, bound, type, prefer)
-        "MmAllocateNonCachedMemory", // MmAllocateNonCachedMemory(size)
-        "MmAllocateMappingAddress", // MmAllocateMappingAddress(size, tag)
-        // --- Windows COM allocation
-        "CoTaskMemAlloc", // CoTaskMemAlloc(size)
-        // --- Solaris/BSD kernel memory allocator
-        "kmem_alloc", // kmem_alloc(size, flags)
-        "kmem_zalloc", // kmem_zalloc(size, flags)
-        // --- OpenSSL memory allocation
-        "CRYPTO_malloc", // CRYPTO_malloc(size_t num, const char *file, int line)
-        "CRYPTO_zalloc", // CRYPTO_zalloc(size_t num, const char *file, int line)
-        "CRYPTO_secure_malloc", // CRYPTO_secure_malloc(size_t num, const char *file, int line)
-        "CRYPTO_secure_zalloc", // CRYPTO_secure_zalloc(size_t num, const char *file, int line)
-        "g_malloc", // g_malloc (n_bytes);
-        "g_try_malloc" // g_try_malloc(n_bytes);
-      ]) and
-    sizeArg = 0
-    or
-    this.hasGlobalName([
-        // --- Windows Memory Management for Windows Drivers
-        "ExAllocatePool", // ExAllocatePool(type, size)
-        "ExAllocatePool2", // ExAllocatePool2(flags, size, tag)
-        "ExAllocatePool3", // ExAllocatePool3(flags, size, tag, extparams, extparamscount)
-        "ExAllocatePoolWithTag", // ExAllocatePool(type, size, tag)
-        "ExAllocatePoolWithTagPriority", // ExAllocatePoolWithTagPriority(type, size, tag, priority)
-        "ExAllocatePoolWithQuota", // ExAllocatePoolWithQuota(type, size)
-        "ExAllocatePoolWithQuotaTag", // ExAllocatePoolWithQuotaTag(type, size, tag)
-        "ExAllocatePoolZero", // ExAllocatePoolZero(type, size, tag)
-        "IoAllocateMdl", // IoAllocateMdl(address, size, flag, flag, irp)
-        "IoAllocateErrorLogEntry", // IoAllocateErrorLogEntry(object, size)
-        // --- Windows Global / Local legacy allocation
-        "LocalAlloc", // LocalAlloc(flags, size)
-        "GlobalAlloc", // GlobalAlloc(flags, size)
-        // --- Windows System Services allocation
-        "VirtualAlloc" // VirtualAlloc(address, size, type, flag)
-      ]) and
-    sizeArg = 1
-    or
-    this.hasGlobalName("HeapAlloc") and // HeapAlloc(heap, flags, size)
-    sizeArg = 2
-    or
-    this.hasGlobalName([
-        // --- Windows Memory Management for Windows Drivers
-        "MmAllocatePagesForMdl", // MmAllocatePagesForMdl(minaddress, maxaddress, skip, size)
-        "MmAllocatePagesForMdlEx", // MmAllocatePagesForMdlEx(minaddress, maxaddress, skip, size, type, flags)
-        "MmAllocateNodePagesForMdlEx" // MmAllocateNodePagesForMdlEx(minaddress, maxaddress, skip, size, type, prefer, flags)
-      ]) and
-    sizeArg = 3
-  }
-
-  override int getSizeArg() { result = sizeArg }
-}
-
-/**
- * An allocation function (such as `alloca`) that does not require a
- * corresponding free (and has an argument for the size in bytes).
- */
-private class AllocaAllocationFunction extends AllocationFunction {
-  int sizeArg;
-
-  AllocaAllocationFunction() {
-    this.hasGlobalName([
-        // --- stack allocation
-        "alloca", // // alloca(size)
-        "__builtin_alloca", // __builtin_alloca(size)
-        "_alloca", // _alloca(size)
-        "_malloca" // _malloca(size)
-      ]) and
-    sizeArg = 0
-  }
-
-  override int getSizeArg() { result = sizeArg }
-
-  override predicate requiresDealloc() { none() }
-}
-
-/**
- * An allocation function (such as `calloc`) that has an argument for the size
- * and another argument for the size of those units (in bytes).
- */
-private class CallocAllocationFunction extends AllocationFunction {
-  int sizeArg;
-  int multArg;
-
-  CallocAllocationFunction() {
-    // --- C library allocation
-    this.hasGlobalOrStdOrBslName("calloc") and // calloc(num, size)
-    sizeArg = 1 and
-    multArg = 0
-  }
-
-  override int getSizeArg() { result = sizeArg }
-
-  override int getSizeMult() { result = multArg }
-}
-
 /**
  * An allocation function (such as `realloc`) that has an argument for the size
  * in bytes, and an argument for an existing pointer that is to be reallocated.
@@ -373,6 +260,63 @@ private class NewArrayAllocationExpr extends AllocationExpr, NewArrayExpr {
   override predicate requiresDealloc() { not exists(this.getPlacementPointer()) }
 }
 
+/**
+ * Holds if `f` is an allocation function according to the
+ * extensible `allocationFunctionModel` predicate.
+ */
+private predicate isAllocationFunctionFromModel(
+  Function f, string namespace, string type, string name
+) {
+  exists(boolean subtypes | allocationFunctionModel(namespace, type, subtypes, name, _, _, _, _) |
+    if type = ""
+    then f.hasQualifiedName(namespace, "", name)
+    else
+      exists(Class c |
+        c.hasQualifiedName(namespace, type) and f.hasQualifiedName(namespace, _, name)
+      |
+        if subtypes = true
+        then f = c.getADerivedClass*().getAMemberFunction()
+        else f = c.getAMemberFunction()
+      )
+  )
+}
+
+/**
+ * An allocation function modeled via the extensible `allocationFunctionModel` predicate.
+ */
+private class AllocationFunctionFromModel extends AllocationFunction {
+  string namespace;
+  string type;
+  string name;
+
+  AllocationFunctionFromModel() { isAllocationFunctionFromModel(this, namespace, type, name) }
+
+  final override int getSizeArg() {
+    exists(string sizeArg |
+      allocationFunctionModel(namespace, type, _, name, sizeArg, _, _, _) and
+      result = sizeArg.toInt()
+    )
+  }
+
+  final override int getSizeMult() {
+    exists(string sizeMult |
+      allocationFunctionModel(namespace, type, _, name, _, sizeMult, _, _) and
+      result = sizeMult.toInt()
+    )
+  }
+
+  final override int getReallocPtrArg() {
+    exists(string reallocPtrArg |
+      allocationFunctionModel(namespace, type, _, name, _, _, reallocPtrArg, _) and
+      result = reallocPtrArg.toInt()
+    )
+  }
+
+  final override predicate requiresDealloc() {
+    allocationFunctionModel(namespace, type, _, name, _, _, _, true)
+  }
+}
+
 private module HeuristicAllocation {
   /** A class that maps an `AllocationExpr` to an `HeuristicAllocationExpr`. */
   private class HeuristicAllocationModeled extends HeuristicAllocationExpr instanceof AllocationExpr

cpp/ql/lib/semmle/code/cpp/models/implementations/Deallocation.qll

@@ -7,61 +7,42 @@
 import semmle.code.cpp.models.interfaces.Deallocation
 
 /**
- * A deallocation function such as `free`.
+ * Holds if `f` is an deallocation function according to the
+ * extensible `deallocationFunctionModel` predicate.
  */
-private class StandardDeallocationFunction extends DeallocationFunction {
+private predicate isDeallocationFunctionFromModel(
-  int freedArg;
+  Function f, string namespace, string type, string name
+) {
+  exists(boolean subtypes | deallocationFunctionModel(namespace, type, subtypes, name, _) |
+    if type = ""
+    then f.hasQualifiedName(namespace, "", name)
+    else
+      exists(Class c |
+        c.hasQualifiedName(namespace, type) and f.hasQualifiedName(namespace, _, name)
+      |
+        if subtypes = true
+        then f = c.getADerivedClass*().getAMemberFunction()
+        else f = c.getAMemberFunction()
+      )
+  )
+}
 
-  StandardDeallocationFunction() {
+/**
-    this.hasGlobalOrStdOrBslName([
+ * A deallocation function modeled via the extensible `deallocationFunctionModel` predicate.
-        // --- C library allocation
+ */
-        "free", "realloc"
+private class DeallocationFunctionFromModel extends DeallocationFunction {
-      ]) and
+  string namespace;
-    freedArg = 0
+  string type;
-    or
+  string name;
-    this.hasGlobalName([
+
-        // --- OpenSSL memory deallocation
+  DeallocationFunctionFromModel() { isDeallocationFunctionFromModel(this, namespace, type, name) }
-        "CRYPTO_free", "CRYPTO_secure_free",
+
-        // --- glib memory deallocation
+  final override int getFreedArg() {
-        "g_free"
+    exists(string freedArg |
-      ]) and
+      deallocationFunctionModel(namespace, type, _, name, freedArg) and
-    freedArg = 0
+      result = freedArg.toInt()
-    or
+    )
-    this.hasGlobalOrStdName([
-        // --- Windows Memory Management for Windows Drivers
-        "ExFreePool", "ExFreePoolWithTag", "ExDeleteTimer", "IoFreeIrp", "IoFreeMdl",
-        "IoFreeErrorLogEntry", "IoFreeWorkItem", "MmFreeContiguousMemory",
-        "MmFreeContiguousMemorySpecifyCache", "MmFreeNonCachedMemory", "MmFreeMappingAddress",
-        "MmFreePagesFromMdl", "MmUnmapReservedMapping", "MmUnmapLockedPages",
-        "NdisFreeGenericObject", "NdisFreeMemory", "NdisFreeMemoryWithTag", "NdisFreeMdl",
-        "NdisFreeNetBufferListPool", "NdisFreeNetBufferPool",
-        // --- Windows Global / Local legacy allocation
-        "LocalFree", "GlobalFree", "LocalReAlloc", "GlobalReAlloc",
-        // --- Windows System Services allocation
-        "VirtualFree",
-        // --- Windows COM allocation
-        "CoTaskMemFree", "CoTaskMemRealloc",
-        // --- Windows Automation
-        "SysFreeString",
-        // --- Solaris/BSD kernel memory allocator
-        "kmem_free"
-      ]) and
-    freedArg = 0
-    or
-    this.hasGlobalOrStdName([
-        // --- Windows Memory Management for Windows Drivers
-        "ExFreeToLookasideListEx", "ExFreeToPagedLookasideList", "ExFreeToNPagedLookasideList",
-        "NdisFreeMemoryWithTagPriority", "StorPortFreeMdl", "StorPortFreePool",
-        // --- NetBSD pool manager
-        "pool_put", "pool_cache_put"
-      ]) and
-    freedArg = 1
-    or
-    this.hasGlobalOrStdName(["HeapFree", "HeapReAlloc"]) and
-    freedArg = 2
   }
-
-  override int getFreedArg() { result = freedArg }
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/models/interfaces/Allocation.qll

@@ -89,6 +89,14 @@ abstract class AllocationFunction extends Function {
   predicate requiresDealloc() { any() }
 }
 
+/**
+ * Holds if an external allocation model exists for the given parameters.
+ */
+extensible predicate allocationFunctionModel(
+  string namespace, string type, boolean subtypes, string name, string sizeArg, string multArg,
+  string reallocPtrArg, boolean requiresDealloc
+);
+
 /**
  * An `operator new` or `operator new[]` function that may be associated with
  * `new` or `new[]` expressions.  Note that `new` and `new[]` are not function

cpp/ql/lib/semmle/code/cpp/models/interfaces/Deallocation.qll

@@ -34,6 +34,13 @@ abstract class DeallocationFunction extends Function {
   int getFreedArg() { none() }
 }
 
+/**
+ * Holds if an external deallocation model exists for the given parameters.
+ */
+extensible predicate deallocationFunctionModel(
+  string namespace, string type, boolean subtypes, string name, string freedArg
+);
+
 /**
  * An `operator delete` or `operator delete[]` function that may be associated
  * with `delete` or `delete[]` expressions.  Note that `delete` and `delete[]`

cpp/ql/lib/semmle/code/cpp/security/flowafterfree/FlowAfterFree.qll

@@ -95,7 +95,7 @@ module FlowFromFree<FlowFromFreeParamSig P> {
         e = any(StoreInstruction store).getDestinationAddress().getUnconvertedResultExpression()
       )
       or
-      n.asExpr() instanceof ArrayExpr
+      [n.asExpr(), n.asIndirectExpr()] instanceof ArrayExpr
     }
   }
 

cpp/ql/src/Security/CWE/CWE-570/IncorrectAllocationErrorHandling.ql

@@ -215,13 +215,18 @@ predicate noThrowInTryBlock(NewOrNewArrayExpr newExpr, BadAllocCatchBlock catchB
  */
 predicate nullCheckInThrowingNew(NewOrNewArrayExpr newExpr, GuardCondition guard) {
   newExpr.getAllocator() instanceof ThrowingAllocator and
-  (
+  // There can be many guard conditions that compares `newExpr` againgst 0.
-    // Handles null comparisons.
+  // For example, for `if(!p)` both `p` and `!p` are guard conditions. To not
-    guard.ensuresEq(globalValueNumber(newExpr).getAnExpr(), any(NullValue null), _, _, _)
+  // produce duplicates results we pick the "first" guard condition according
-    or
+  // to some arbitrary ordering (i.e., location information). This means `!p` is the
-    // Handles `if(ptr)` and `if(!ptr)` cases.
+  // element that we use to construct the alert.
-    guard = globalValueNumber(newExpr).getAnExpr()
+  guard =
-  )
+    min(GuardCondition gc, int startline, int startcolumn, int endline, int endcolumn |
+      gc.comparesEq(globalValueNumber(newExpr).getAnExpr(), 0, _, _) and
+      gc.getLocation().hasLocationInfo(_, startline, startcolumn, endline, endcolumn)
+    |
+      gc order by startline, startcolumn, endline, endcolumn
+    )
 }
 
 from NewOrNewArrayExpr newExpr, Element element, string msg, string elementString

cpp/ql/test/header-variant-tests/iquote/iquote.c

@@ -3,4 +3,4 @@
 #include "b.h"
 static int has_angle_b = __has_include(<b.h>);
 
-// semmle-extractor-options: -I${testdir}/dir2 -iquote ${testdir}/dir1 --edg --clang
+// semmle-extractor-options: -I${testdir}/dir2 -iquote ${testdir}/dir1 --clang

cpp/ql/test/library-tests/CPP-207/options

@@ -1 +1 @@
-semmle-extractor-options: --edg --microsoft
+semmle-extractor-options: --microsoft

cpp/ql/test/library-tests/arguments/arguments.expected

@@ -15,8 +15,6 @@
 | arguments.c | 15 | --edg |
 | arguments.c | 16 | __CODEQL_TEST__ |
 | arguments.c | 17 | --gcc |
-| arguments.c | 18 | --predefined_macros |
+| arguments.c | 18 | -w |
-| arguments.c | 19 | <tools>/qltest/predefined_macros |
+| arguments.c | 19 | -Werror |
-| arguments.c | 20 | -w |
+| arguments.c | 20 | arguments.c |
-| arguments.c | 21 | -Werror |
-| arguments.c | 22 | arguments.c |

cpp/ql/test/library-tests/arguments/arguments.ql

@@ -4,8 +4,5 @@ from Compilation c, int i, string s
 // Skip the extractor name; it'll vary depending on platform
 where
   i > 0 and
-  s =
+  s = c.getArgument(i).replaceAll("\\", "/")
-    c.getArgument(i)
-        .replaceAll("\\", "/")
-        .regexpReplaceAll(".*(/qltest/predefined_macros)", "<tools>$1")
 select c.getAFileCompiled().toString(), i, s

cpp/ql/test/library-tests/attributes/availability/options

@@ -1 +1 @@
-semmle-extractor-options: --edg --clang
+semmle-extractor-options: --clang

cpp/ql/test/library-tests/attributes/routine_attributes/arguments.expected

@@ -1,4 +1,8 @@
 | declspec.cpp:4:23:4:43 | Use fatal() instead | declspec.cpp:4:59:4:62 | exit | declspec.cpp:4:12:4:21 | deprecated | Use fatal() instead |
+| routine_attributes2.cpp:5:6:5:11 | hidden | routine_attributes2.cpp:5:13:5:21 | a_routine | routine_attributes2.cpp:5:6:5:11 | visibility | hidden |
+| routine_attributes2.cpp:5:6:5:11 | hidden | routine_attributes2.cpp:5:13:5:21 | a_routine | routine_attributes2.cpp:5:6:5:11 | visibility | hidden |
+| routine_attributes2.h:3:6:3:11 | hidden | routine_attributes2.cpp:5:13:5:21 | a_routine | routine_attributes2.h:3:6:3:11 | visibility | hidden |
+| routine_attributes2.h:3:6:3:11 | hidden | routine_attributes2.cpp:5:13:5:21 | a_routine | routine_attributes2.h:3:6:3:11 | visibility | hidden |
 | routine_attributes.c:3:53:3:59 | dummy | routine_attributes.c:3:12:3:24 | named_weakref | routine_attributes.c:3:44:3:50 | weakref | dummy |
 | routine_attributes.c:4:62:4:68 | dummy | routine_attributes.c:4:12:4:26 | aliased_weakref | routine_attributes.c:4:55:4:59 | alias | dummy |
 | routine_attributes.c:6:49:6:55 | dummy | routine_attributes.c:6:12:6:22 | plain_alias | routine_attributes.c:6:42:6:46 | alias | dummy |

cpp/ql/test/library-tests/attributes/routine_attributes/routine_attributes.expected

@@ -18,6 +18,10 @@
 | header_export.cpp:14:16:14:26 | myFunction4 | header_export.cpp:14:1:14:9 | dllexport |
 | header_export.cpp:18:6:18:16 | myFunction5 | header.h:10:2:10:10 | dllexport |
 | header_export.cpp:18:6:18:16 | myFunction5 | header.h:10:2:10:10 | dllimport |
+| routine_attributes2.cpp:5:13:5:21 | a_routine | routine_attributes2.cpp:5:6:5:11 | visibility |
+| routine_attributes2.cpp:5:13:5:21 | a_routine | routine_attributes2.cpp:5:6:5:11 | visibility |
+| routine_attributes2.cpp:5:13:5:21 | a_routine | routine_attributes2.h:3:6:3:11 | visibility |
+| routine_attributes2.cpp:5:13:5:21 | a_routine | routine_attributes2.h:3:6:3:11 | visibility |
 | routine_attributes.c:3:12:3:24 | named_weakref | routine_attributes.c:3:44:3:50 | weakref |
 | routine_attributes.c:4:12:4:26 | aliased_weakref | routine_attributes.c:4:46:4:52 | weakref |
 | routine_attributes.c:4:12:4:26 | aliased_weakref | routine_attributes.c:4:55:4:59 | alias |

cpp/ql/test/library-tests/attributes/routine_attributes/routine_attributes2.cpp

@@ -0,0 +1,7 @@
+#define HIDDEN __attribute__((visibility("hidden")))
+
+#include "routine_attributes2.h"
+
+void HIDDEN a_routine() {
+    return;
+}

cpp/ql/test/library-tests/attributes/routine_attributes/routine_attributes2.h

@@ -0,0 +1,3 @@
+#pragma once
+
+void HIDDEN a_routine();

cpp/ql/test/library-tests/attributes/routine_attributes/routine_attributes3.cpp

@@ -0,0 +1,3 @@
+#define HIDDEN __attribute__((visibility("hidden")))
+
+#include "routine_attributes2.h"

cpp/ql/test/library-tests/attributes/type_attributes/arguments.expected

@@ -1,3 +1,6 @@
+| type_attributes2.cpp:5:14:5:20 | a_class | type_attributes2.cpp:5:7:5:12 | visibility | type_attributes2.cpp:5:7:5:12 | hidden |
+| type_attributes2.cpp:5:14:5:20 | a_class | type_attributes2.h:3:7:3:12 | visibility | type_attributes2.h:3:7:3:12 | hidden |
+| type_attributes2.cpp:5:14:5:20 | a_class | type_attributes2.h:3:7:3:12 | visibility | type_attributes2.h:3:7:3:12 | hidden |
 | type_attributes_ms.cpp:4:67:4:75 | IDispatch | type_attributes_ms.cpp:4:19:4:22 | uuid | type_attributes_ms.cpp:4:24:4:63 | {00020400-0000-0000-c000-000000000046} |
 | type_attributes_ms.cpp:5:30:5:33 | Str1 | type_attributes_ms.cpp:5:12:5:16 | align | type_attributes_ms.cpp:5:18:5:19 | 32 |
 | type_attributes_ms.cpp:6:55:6:62 | IUnknown | type_attributes_ms.cpp:6:2:6:2 | uuid | type_attributes_ms.cpp:6:2:6:2 | 00000000-0000-0000-c000-000000000046 |

cpp/ql/test/library-tests/attributes/type_attributes/type_attributes.expected

@@ -1,4 +1,7 @@
 | file://:0:0:0:0 | short __attribute((__may_alias__)) | type_attributes.c:25:30:25:42 | may_alias |
+| type_attributes2.cpp:5:14:5:20 | a_class | type_attributes2.cpp:5:7:5:12 | visibility |
+| type_attributes2.cpp:5:14:5:20 | a_class | type_attributes2.h:3:7:3:12 | visibility |
+| type_attributes2.cpp:5:14:5:20 | a_class | type_attributes2.h:3:7:3:12 | visibility |
 | type_attributes.c:5:36:5:51 | my_packed_struct | type_attributes.c:5:23:5:32 | packed |
 | type_attributes.c:10:54:10:54 | (unnamed class/struct/union) | type_attributes.c:10:30:10:50 | transparent_union |
 | type_attributes.c:16:54:16:54 | (unnamed class/struct/union) | type_attributes.c:16:30:16:50 | transparent_union |

cpp/ql/test/library-tests/attributes/type_attributes/type_attributes2.cpp

@@ -0,0 +1,6 @@
+#define HIDDEN __attribute__((visibility("hidden")))
+
+#include "type_attributes2.h"
+
+class HIDDEN a_class {
+};

cpp/ql/test/library-tests/attributes/type_attributes/type_attributes2.h

@@ -0,0 +1,3 @@
+#pragma once
+
+class HIDDEN a_class;

cpp/ql/test/library-tests/attributes/type_attributes/type_attributes3.cpp

@@ -0,0 +1,3 @@
+#define HIDDEN __attribute__((visibility("hidden")))
+
+#include "type_attributes2.h"

cpp/ql/test/library-tests/attributes/var_attributes/var_attributes.expected

@@ -6,6 +6,10 @@
 | ms_var_attributes.cpp:12:42:12:46 | field | ms_var_attributes.cpp:12:14:12:21 | property |
 | ms_var_attributes.cpp:20:34:20:37 | pBuf | ms_var_attributes.cpp:20:12:20:12 | SAL_volatile |
 | ms_var_attributes.h:5:22:5:27 | myInt3 | ms_var_attributes.h:5:1:5:9 | dllexport |
+| var_attributes2.cpp:5:12:5:21 | a_variable | var_attributes2.cpp:5:5:5:10 | visibility |
+| var_attributes2.cpp:5:12:5:21 | a_variable | var_attributes2.cpp:5:5:5:10 | visibility |
+| var_attributes2.cpp:5:12:5:21 | a_variable | var_attributes2.h:3:12:3:17 | visibility |
+| var_attributes2.cpp:5:12:5:21 | a_variable | var_attributes2.h:3:12:3:17 | visibility |
 | var_attributes.c:1:12:1:19 | weak_var | var_attributes.c:1:36:1:39 | weak |
 | var_attributes.c:2:12:2:22 | weakref_var | var_attributes.c:2:39:2:45 | weakref |
 | var_attributes.c:3:12:3:19 | used_var | var_attributes.c:3:36:3:39 | used |

cpp/ql/test/library-tests/attributes/var_attributes/var_attributes2.cpp

@@ -0,0 +1,5 @@
+#define HIDDEN __attribute__((visibility("hidden")))
+
+#include "var_attributes2.h"
+
+int HIDDEN a_variable;

cpp/ql/test/library-tests/attributes/var_attributes/var_attributes2.h

@@ -0,0 +1,3 @@
+#pragma once
+
+extern int HIDDEN a_variable;

cpp/ql/test/library-tests/attributes/var_attributes/var_attributes3.cpp

@@ -0,0 +1,3 @@
+#define HIDDEN __attribute__((visibility("hidden")))
+
+#include "var_attributes2.h"

cpp/ql/test/library-tests/clang_builtin_macros/addressof.c

@@ -1,4 +1,4 @@
-// semmle-extractor-options: --edg --clang
+// semmle-extractor-options: --clang
 
 int x = 0;
 

cpp/ql/test/library-tests/clang_builtin_macros/extended.cpp

@@ -1,4 +1,4 @@
-// semmle-extractor-options: --edg --clang --edg --c++11 --edg --nullptr
+// semmle-extractor-options: --clang --edg --c++11 --edg --nullptr
 
 static int has_nullptr_f = __has_feature(cxx_nullptr);
 static int has_nullptr_e = __has_extension(cxx_nullptr);

cpp/ql/test/library-tests/clang_builtin_macros/options

@@ -1 +1 @@
-semmle-extractor-options: --edg --clang
+semmle-extractor-options: --clang

cpp/ql/test/library-tests/clang_builtin_macros/test.cpp

@@ -1,7 +1,7 @@
 // For the canonical behaviour, run: clang -E -w test.cpp
 #define __builtin_TRAP __builtin_trap
 #define BAR "bar.h"
-// semmle-extractor-options: --edg --clang --expect_errors
+// semmle-extractor-options: --clang --expect_errors
 #if defined(__has_include)
 static int has_include = 1;
 #else

cpp/ql/test/library-tests/clang_ms/options

@@ -1 +1 @@
-semmle-extractor-options: --edg --clang --edg --ms_extensions
+semmle-extractor-options: --clang --edg --ms_extensions

cpp/ql/test/library-tests/controlflow/guards/Guards.expected

@@ -45,3 +45,7 @@
 | test.cpp:122:9:122:9 | b |
 | test.cpp:125:13:125:20 | ! ... |
 | test.cpp:125:14:125:17 | call to safe |
+| test.cpp:131:6:131:21 | call to __builtin_expect |
+| test.cpp:135:6:135:21 | call to __builtin_expect |
+| test.cpp:141:6:141:21 | call to __builtin_expect |
+| test.cpp:145:6:145:21 | call to __builtin_expect |

cpp/ql/test/library-tests/controlflow/guards/GuardsCompare.expected

@@ -164,10 +164,46 @@
 | 126 | call to test3_condition != 0 when ... && ... is true |
 | 126 | call to test3_condition != 0 when call to test3_condition is true |
 | 126 | call to test3_condition == 0 when call to test3_condition is false |
+| 131 | ... + ... != a+0 when call to __builtin_expect is false |
+| 131 | ... + ... == a+0 when call to __builtin_expect is true |
+| 131 | a != ... + ...+0 when call to __builtin_expect is false |
+| 131 | a != b+42 when call to __builtin_expect is false |
+| 131 | a == ... + ...+0 when call to __builtin_expect is true |
+| 131 | a == b+42 when call to __builtin_expect is true |
 | 131 | b != 0 when b is true |
+| 131 | b != a+-42 when call to __builtin_expect is false |
 | 131 | b == 0 when b is false |
+| 131 | b == a+-42 when call to __builtin_expect is true |
+| 131 | call to __builtin_expect != 0 when call to __builtin_expect is true |
+| 131 | call to __builtin_expect == 0 when call to __builtin_expect is false |
+| 135 | ... + ... != a+0 when call to __builtin_expect is true |
+| 135 | ... + ... == a+0 when call to __builtin_expect is false |
+| 135 | a != ... + ...+0 when call to __builtin_expect is true |
+| 135 | a != b+42 when call to __builtin_expect is true |
+| 135 | a == ... + ...+0 when call to __builtin_expect is false |
+| 135 | a == b+42 when call to __builtin_expect is false |
+| 135 | b != a+-42 when call to __builtin_expect is true |
+| 135 | b == a+-42 when call to __builtin_expect is false |
+| 135 | call to __builtin_expect != 0 when call to __builtin_expect is true |
+| 135 | call to __builtin_expect == 0 when call to __builtin_expect is false |
 | 137 | 0 != 0 when 0 is true |
 | 137 | 0 == 0 when 0 is false |
+| 141 | 42 != a+0 when call to __builtin_expect is false |
+| 141 | 42 == a+0 when call to __builtin_expect is true |
+| 141 | a != 42 when call to __builtin_expect is false |
+| 141 | a != 42+0 when call to __builtin_expect is false |
+| 141 | a == 42 when call to __builtin_expect is true |
+| 141 | a == 42+0 when call to __builtin_expect is true |
+| 141 | call to __builtin_expect != 0 when call to __builtin_expect is true |
+| 141 | call to __builtin_expect == 0 when call to __builtin_expect is false |
+| 145 | 42 != a+0 when call to __builtin_expect is true |
+| 145 | 42 == a+0 when call to __builtin_expect is false |
+| 145 | a != 42 when call to __builtin_expect is true |
+| 145 | a != 42+0 when call to __builtin_expect is true |
+| 145 | a == 42 when call to __builtin_expect is false |
+| 145 | a == 42+0 when call to __builtin_expect is false |
+| 145 | call to __builtin_expect != 0 when call to __builtin_expect is true |
+| 145 | call to __builtin_expect == 0 when call to __builtin_expect is false |
 | 146 | ! ... != 0 when ! ... is true |
 | 146 | ! ... == 0 when ! ... is false |
 | 146 | x != 0 when ! ... is false |

cpp/ql/test/library-tests/controlflow/guards/GuardsControl.expected

@@ -104,3 +104,7 @@
 | test.cpp:122:9:122:9 | b | true | 125 | 125 |
 | test.cpp:125:13:125:20 | ! ... | true | 125 | 125 |
 | test.cpp:125:14:125:17 | call to safe | false | 125 | 125 |
+| test.cpp:131:6:131:21 | call to __builtin_expect | true | 131 | 132 |
+| test.cpp:135:6:135:21 | call to __builtin_expect | true | 135 | 136 |
+| test.cpp:141:6:141:21 | call to __builtin_expect | true | 141 | 142 |
+| test.cpp:145:6:145:21 | call to __builtin_expect | true | 145 | 146 |

cpp/ql/test/library-tests/controlflow/guards/GuardsEnsure.expected

@@ -159,6 +159,18 @@ binary
 | test.cpp:105:6:105:14 | ... != ... | test.cpp:105:11:105:14 | 0.0 | != | test.cpp:105:6:105:6 | f | 0 | 105 | 106 |
 | test.cpp:111:6:111:14 | ... != ... | test.cpp:111:6:111:6 | i | != | test.cpp:111:11:111:14 | 0.0 | 0 | 111 | 112 |
 | test.cpp:111:6:111:14 | ... != ... | test.cpp:111:11:111:14 | 0.0 | != | test.cpp:111:6:111:6 | i | 0 | 111 | 112 |
+| test.cpp:131:6:131:21 | call to __builtin_expect | test.cpp:131:23:131:23 | a | == | test.cpp:131:28:131:28 | b | 42 | 131 | 132 |
+| test.cpp:131:6:131:21 | call to __builtin_expect | test.cpp:131:23:131:23 | a | == | test.cpp:131:28:131:33 | ... + ... | 0 | 131 | 132 |
+| test.cpp:131:6:131:21 | call to __builtin_expect | test.cpp:131:28:131:28 | b | == | test.cpp:131:23:131:23 | a | -42 | 131 | 132 |
+| test.cpp:131:6:131:21 | call to __builtin_expect | test.cpp:131:28:131:33 | ... + ... | == | test.cpp:131:23:131:23 | a | 0 | 131 | 132 |
+| test.cpp:135:6:135:21 | call to __builtin_expect | test.cpp:135:23:135:23 | a | != | test.cpp:135:28:135:28 | b | 42 | 135 | 136 |
+| test.cpp:135:6:135:21 | call to __builtin_expect | test.cpp:135:23:135:23 | a | != | test.cpp:135:28:135:33 | ... + ... | 0 | 135 | 136 |
+| test.cpp:135:6:135:21 | call to __builtin_expect | test.cpp:135:28:135:28 | b | != | test.cpp:135:23:135:23 | a | -42 | 135 | 136 |
+| test.cpp:135:6:135:21 | call to __builtin_expect | test.cpp:135:28:135:33 | ... + ... | != | test.cpp:135:23:135:23 | a | 0 | 135 | 136 |
+| test.cpp:141:6:141:21 | call to __builtin_expect | test.cpp:141:23:141:23 | a | == | test.cpp:141:28:141:29 | 42 | 0 | 141 | 142 |
+| test.cpp:141:6:141:21 | call to __builtin_expect | test.cpp:141:28:141:29 | 42 | == | test.cpp:141:23:141:23 | a | 0 | 141 | 142 |
+| test.cpp:145:6:145:21 | call to __builtin_expect | test.cpp:145:23:145:23 | a | != | test.cpp:145:28:145:29 | 42 | 0 | 145 | 146 |
+| test.cpp:145:6:145:21 | call to __builtin_expect | test.cpp:145:28:145:29 | 42 | != | test.cpp:145:23:145:23 | a | 0 | 145 | 146 |
 unary
 | test.c:7:9:7:13 | ... > ... | test.c:7:9:7:9 | x | < | 1 | 10 | 11 |
 | test.c:7:9:7:13 | ... > ... | test.c:7:9:7:9 | x | >= | 1 | 7 | 9 |
@@ -270,3 +282,9 @@ unary
 | test.cpp:122:9:122:9 | b | test.cpp:122:9:122:9 | b | != | 0 | 125 | 125 |
 | test.cpp:125:13:125:20 | ! ... | test.cpp:125:13:125:20 | ! ... | != | 0 | 125 | 125 |
 | test.cpp:125:14:125:17 | call to safe | test.cpp:125:14:125:17 | call to safe | == | 0 | 125 | 125 |
+| test.cpp:131:6:131:21 | call to __builtin_expect | test.cpp:131:6:131:21 | call to __builtin_expect | != | 0 | 131 | 132 |
+| test.cpp:135:6:135:21 | call to __builtin_expect | test.cpp:135:6:135:21 | call to __builtin_expect | != | 0 | 135 | 136 |
+| test.cpp:141:6:141:21 | call to __builtin_expect | test.cpp:141:6:141:21 | call to __builtin_expect | != | 0 | 141 | 142 |
+| test.cpp:141:6:141:21 | call to __builtin_expect | test.cpp:141:23:141:23 | a | == | 42 | 141 | 142 |
+| test.cpp:145:6:145:21 | call to __builtin_expect | test.cpp:145:6:145:21 | call to __builtin_expect | != | 0 | 145 | 146 |
+| test.cpp:145:6:145:21 | call to __builtin_expect | test.cpp:145:23:145:23 | a | != | 42 | 145 | 146 |

cpp/ql/test/library-tests/controlflow/guards/test.cpp

@@ -125,4 +125,24 @@ void test(bool b)
         if (!safe(x)) return;
     }
     use(x);
+}
+
+void binary_test_builtin_expected(int a, int b) {
+  if(__builtin_expect(a == b + 42, 0)) {
+      use(a);
+  }
+
+  if(__builtin_expect(a != b + 42, 0)) {
+      use(a);
+  }
+}
+
+void unary_test_builtin_expected(int a) {
+  if(__builtin_expect(a == 42, 0)) {
+      use(a);
+  }
+
+  if(__builtin_expect(a != 42, 0)) {
+      use(a);
+  }
 }

cpp/ql/test/library-tests/dataflow/dataflow-edge-tests/additionalEdges.expected

@@ -1,6 +1,6 @@
-WARNING: Module DataFlow has been deprecated and may be removed in future (additionalEdges.ql:31,6-14)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (additionalEdges.ql:31,6-14)
-WARNING: Module DataFlow has been deprecated and may be removed in future (additionalEdges.ql:31,31-39)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (additionalEdges.ql:31,31-39)
-WARNING: Module DataFlow has been deprecated and may be removed in future (additionalEdges.ql:32,7-15)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (additionalEdges.ql:32,7-15)
 | tryExcept.c:7:7:7:7 | x | tryExcept.c:14:10:14:10 | x |
 | tryExcept.c:7:13:7:14 | 0 | tryExcept.c:10:9:10:9 | y |
 | tryExcept.c:10:9:10:9 | y | tryExcept.c:10:5:10:9 | ... = ... |

cpp/ql/test/library-tests/dataflow/dataflow-edge-tests/standardEdges.expected

@@ -1,5 +1,5 @@
-WARNING: Module DataFlow has been deprecated and may be removed in future (standardEdges.ql:4,6-14)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (standardEdges.ql:4,6-14)
-WARNING: Module DataFlow has been deprecated and may be removed in future (standardEdges.ql:4,31-39)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (standardEdges.ql:4,31-39)
-WARNING: Module DataFlow has been deprecated and may be removed in future (standardEdges.ql:5,7-15)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (standardEdges.ql:5,7-15)
 | tryExcept.c:7:13:7:14 | 0 | tryExcept.c:10:9:10:9 | y |
 | tryExcept.c:10:9:10:9 | y | tryExcept.c:10:5:10:9 | ... = ... |

cpp/ql/test/library-tests/dataflow/dataflow-tests/TestBase.qll

@@ -151,6 +151,9 @@ module IRTest {
         or
         call.getTarget().getName() = "indirect_sink" and
         sink.asIndirectExpr() = e
+        or
+        call.getTarget().getName() = "indirect_sink_const_ref" and
+        sink.asIndirectExpr() = e
       )
     }
 

cpp/ql/test/library-tests/dataflow/dataflow-tests/clang.cpp

@@ -1,4 +1,4 @@
-// semmle-extractor-options: --edg --clang
+// semmle-extractor-options: --clang
 
 int source();
 void sink(int); void sink(const int *); void sink(int **); void indirect_sink(...);

cpp/ql/test/library-tests/dataflow/dataflow-tests/has-parameter-flow-out.expected

@@ -1,3 +1,3 @@
-WARNING: Module DataFlow has been deprecated and may be removed in future (has-parameter-flow-out.ql:5,18-61)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (has-parameter-flow-out.ql:5,18-61)
 testFailures
 failures

cpp/ql/test/library-tests/dataflow/dataflow-tests/localFlow.expected

@@ -1,6 +1,6 @@
-WARNING: Module DataFlow has been deprecated and may be removed in future (localFlow.ql:4,6-14)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (localFlow.ql:4,6-14)
-WARNING: Module DataFlow has been deprecated and may be removed in future (localFlow.ql:4,31-39)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (localFlow.ql:4,31-39)
-WARNING: Module DataFlow has been deprecated and may be removed in future (localFlow.ql:6,3-11)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (localFlow.ql:6,3-11)
 | example.c:15:37:15:37 | b | example.c:15:37:15:37 | b |
 | example.c:15:37:15:37 | b | example.c:19:6:19:6 | b |
 | example.c:15:44:15:46 | pos | example.c:24:24:24:26 | pos |

cpp/ql/test/library-tests/dataflow/dataflow-tests/test-number-of-outnodes.expected

@@ -1,3 +1,3 @@
-WARNING: Module DataFlow has been deprecated and may be removed in future (test-number-of-outnodes.ql:5,18-61)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (test-number-of-outnodes.ql:5,18-61)
 failures
 testFailures

cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected

@@ -1,5 +1,5 @@
-WARNING: Module DataFlow has been deprecated and may be removed in future (test-source-sink.ql:3,25-42)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (test-source-sink.ql:3,25-42)
-WARNING: Module DataFlow has been deprecated and may be removed in future (test-source-sink.ql:3,57-74)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (test-source-sink.ql:3,57-74)
 astFlow
 | BarrierGuard.cpp:5:19:5:24 | source | BarrierGuard.cpp:9:10:9:15 | source |
 | BarrierGuard.cpp:13:17:13:22 | source | BarrierGuard.cpp:15:10:15:15 | source |
@@ -313,6 +313,7 @@ irFlow
 | test.cpp:1021:18:1021:32 | *call to indirect_source | test.cpp:1027:19:1027:28 | *translated |
 | test.cpp:1021:18:1021:32 | *call to indirect_source | test.cpp:1031:19:1031:28 | *translated |
 | test.cpp:1045:14:1045:19 | call to source | test.cpp:1046:7:1046:10 | * ... |
+| test.cpp:1081:27:1081:34 | call to source | test.cpp:1081:27:1081:34 | call to source |
 | true_upon_entry.cpp:9:11:9:16 | call to source | true_upon_entry.cpp:13:8:13:8 | x |
 | true_upon_entry.cpp:17:11:17:16 | call to source | true_upon_entry.cpp:21:8:21:8 | x |
 | true_upon_entry.cpp:27:9:27:14 | call to source | true_upon_entry.cpp:29:8:29:8 | x |

cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp

@@ -1073,3 +1073,10 @@ void single_object_in_both_cases(bool b, int x, int y) {
   *p = 0;
   sink(*p); // clean
 }
+
+template<typename T>
+void indirect_sink_const_ref(const T&);
+
+void test_temp_with_conversion_from_materialization() {
+  indirect_sink_const_ref(source()); // $ ir MISSING: ast
+}

cpp/ql/test/library-tests/dataflow/fields/partial-definition-diff.expected

@@ -1,4 +1,4 @@
-WARNING: Module DataFlow has been deprecated and may be removed in future (partial-definition-diff.ql:7,8-51)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (partial-definition-diff.ql:7,8-51)
 | A.cpp:25:13:25:13 | c | AST only |
 | A.cpp:27:28:27:28 | c | AST only |
 | A.cpp:28:23:28:26 | this | IR only |

cpp/ql/test/library-tests/dataflow/fields/partial-definition.expected

@@ -1,4 +1,4 @@
-WARNING: Module DataFlow has been deprecated and may be removed in future (partial-definition.ql:6,8-51)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (partial-definition.ql:6,8-51)
 | A.cpp:25:7:25:10 | this |
 | A.cpp:25:13:25:13 | c |
 | A.cpp:27:22:27:25 | this |

cpp/ql/test/library-tests/dataflow/smart-pointers-taint/taint.expected

@@ -1,6 +1,6 @@
-WARNING: Module DataFlow has been deprecated and may be removed in future (taint.ql:6,48-56)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (taint.ql:6,48-56)
-WARNING: Module DataFlow has been deprecated and may be removed in future (taint.ql:7,24-32)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (taint.ql:7,24-32)
-WARNING: Module DataFlow has been deprecated and may be removed in future (taint.ql:11,22-30)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (taint.ql:11,22-30)
-WARNING: Module TaintTracking has been deprecated and may be removed in future (taint.ql:19,20-33)
+WARNING: module 'TaintTracking' has been deprecated and may be removed in future (taint.ql:19,20-33)
 failures
 testFailures

cpp/ql/test/library-tests/dataflow/taint-tests/localTaint.expected

@@ -1,7 +1,7 @@
-WARNING: Module DataFlow has been deprecated and may be removed in future (localTaint.ql:4,6-14)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (localTaint.ql:4,6-14)
-WARNING: Module DataFlow has been deprecated and may be removed in future (localTaint.ql:4,31-39)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (localTaint.ql:4,31-39)
-WARNING: Module DataFlow has been deprecated and may be removed in future (localTaint.ql:7,6-14)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (localTaint.ql:7,6-14)
-WARNING: Module TaintTracking has been deprecated and may be removed in future (localTaint.ql:6,3-16)
+WARNING: module 'TaintTracking' has been deprecated and may be removed in future (localTaint.ql:6,3-16)
 | arrayassignment.cpp:9:9:9:10 | 0 | arrayassignment.cpp:10:14:10:14 | x |  |
 | arrayassignment.cpp:9:9:9:10 | 0 | arrayassignment.cpp:11:15:11:15 | x |  |
 | arrayassignment.cpp:9:9:9:10 | 0 | arrayassignment.cpp:12:13:12:13 | x |  |

cpp/ql/test/library-tests/dataflow/taint-tests/taint.expected

@@ -1,7 +1,7 @@
-WARNING: Module DataFlow has been deprecated and may be removed in future (taint.ql:46,45-53)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (taint.ql:46,45-53)
-WARNING: Module DataFlow has been deprecated and may be removed in future (taint.ql:47,24-32)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (taint.ql:47,24-32)
-WARNING: Module DataFlow has been deprecated and may be removed in future (taint.ql:61,22-30)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (taint.ql:61,22-30)
-WARNING: Module DataFlow has been deprecated and may be removed in future (taint.ql:68,25-33)
+WARNING: module 'DataFlow' has been deprecated and may be removed in future (taint.ql:68,25-33)
-WARNING: Module TaintTracking has been deprecated and may be removed in future (taint.ql:73,20-33)
+WARNING: module 'TaintTracking' has been deprecated and may be removed in future (taint.ql:73,20-33)
 testFailures
 failures

cpp/ql/test/library-tests/declspec/guard/test.cpp

@@ -1,4 +1,4 @@
-// semmle-extractor-options: --edg --microsoft
+// semmle-extractor-options: --microsoft
 
 void f(__declspec(guard(overflow)) size_t length) {
 }

cpp/ql/test/library-tests/specifiers2/cpp20.cpp

@@ -1,4 +1,4 @@
-// semmle-extractor-options: --edg --clang --edg --c++20
+// semmle-extractor-options: --clang --edg --c++20
 
 namespace cpp20 {
 

cpp/ql/test/library-tests/structs/compatible_c/c1_gnu.c

@@ -7,4 +7,4 @@ struct Kiwi {
 struct Lemon {
   unsigned int __attribute__ ((vector_size (16))) lemon_x;
 };
-// semmle-extractor-options: --edg --c99 --edg --clang --edg --clang_vector_types --gnu_version 40700
+// semmle-extractor-options: --edg --c99 --clang --edg --clang_vector_types --gnu_version 40700

cpp/ql/test/library-tests/structs/compatible_c/c2_gnu.c

@@ -7,4 +7,4 @@ struct Kiwi {
 struct Lemon {
   signed int __attribute__ ((vector_size (16))) lemon_x;
 };
-// semmle-extractor-options: --edg --c99 --edg --clang --edg --clang_vector_types --gnu_version 40700
+// semmle-extractor-options: --edg --c99 --clang --edg --clang_vector_types --gnu_version 40700

cpp/ql/test/library-tests/templates/CPP-223/test.cpp

@@ -1,4 +1,4 @@
-// semmle-extractor-options: --edg --clang --edg --ms_extensions
+// semmle-extractor-options: --clang --edg --ms_extensions
 
 template <int a> class b {
   template <bool> struct c;

cpp/ql/test/library-tests/vector_types/options

@@ -1 +1 @@
-semmle-extractor-options: --edg --clang --edg --clang_builtin_functions --edg --clang_vector_types --gnu_version 40600
+semmle-extractor-options: --clang --edg --clang_builtin_functions --edg --clang_vector_types --gnu_version 40600

cpp/ql/test/library-tests/vector_types/vector_types.cpp

@@ -1,4 +1,4 @@
-// semmle-extractor-options: --edg --clang --edg --clang_version --edg 30801
+// semmle-extractor-options: --clang --edg --clang_version --edg 30801
 // Compilable with: clang --std=c++0x -msse4.1 vector_types.cpp
 // (some bits also compilable with gcc)
 int printf(...);

cpp/ql/test/query-tests/Critical/MemoryFreed/DoubleFree.expected

@@ -13,21 +13,6 @@ edges
 | test_free.cpp:207:10:207:10 | pointer to free output argument | test_free.cpp:209:10:209:10 | a | provenance |  |
 | test_free.cpp:301:12:301:14 | pointer to g_free output argument | test_free.cpp:302:12:302:14 | buf | provenance |  |
 | test_free.cpp:319:16:319:16 | pointer to operator delete output argument | test_free.cpp:322:12:322:12 | a | provenance |  |
-| test_free.cpp:343:12:343:24 | *access to array [post update] [ptr] | test_free.cpp:344:12:344:24 | *access to array [ptr] | provenance |  |
-| test_free.cpp:343:12:343:24 | *access to array [post update] [ptr] | test_free.cpp:345:12:345:24 | *access to array [ptr] | provenance |  |
-| test_free.cpp:343:12:343:24 | *access to array [post update] [ptr] | test_free.cpp:346:12:346:24 | *access to array [ptr] | provenance |  |
-| test_free.cpp:343:26:343:28 | pointer to operator delete output argument | test_free.cpp:343:12:343:24 | *access to array [post update] [ptr] | provenance |  |
-| test_free.cpp:344:12:344:24 | *access to array [post update] [ptr] | test_free.cpp:345:12:345:24 | *access to array [ptr] | provenance |  |
-| test_free.cpp:344:12:344:24 | *access to array [post update] [ptr] | test_free.cpp:346:12:346:24 | *access to array [ptr] | provenance |  |
-| test_free.cpp:344:12:344:24 | *access to array [ptr] | test_free.cpp:344:26:344:28 | ptr | provenance |  |
-| test_free.cpp:344:26:344:28 | pointer to operator delete output argument | test_free.cpp:344:12:344:24 | *access to array [post update] [ptr] | provenance |  |
-| test_free.cpp:345:12:345:24 | *access to array [post update] [ptr] | test_free.cpp:346:12:346:24 | *access to array [ptr] | provenance |  |
-| test_free.cpp:345:12:345:24 | *access to array [ptr] | test_free.cpp:345:26:345:28 | ptr | provenance |  |
-| test_free.cpp:345:12:345:24 | *access to array [ptr] | test_free.cpp:345:26:345:28 | ptr | provenance |  |
-| test_free.cpp:345:26:345:28 | pointer to operator delete output argument | test_free.cpp:345:12:345:24 | *access to array [post update] [ptr] | provenance |  |
-| test_free.cpp:346:12:346:24 | *access to array [ptr] | test_free.cpp:346:26:346:28 | ptr | provenance |  |
-| test_free.cpp:346:12:346:24 | *access to array [ptr] | test_free.cpp:346:26:346:28 | ptr | provenance |  |
-| test_free.cpp:346:12:346:24 | *access to array [ptr] | test_free.cpp:346:26:346:28 | ptr | provenance |  |
 nodes
 | test_free.cpp:11:10:11:10 | pointer to free output argument | semmle.label | pointer to free output argument |
 | test_free.cpp:14:10:14:10 | a | semmle.label | a |
@@ -57,24 +42,6 @@ nodes
 | test_free.cpp:302:12:302:14 | buf | semmle.label | buf |
 | test_free.cpp:319:16:319:16 | pointer to operator delete output argument | semmle.label | pointer to operator delete output argument |
 | test_free.cpp:322:12:322:12 | a | semmle.label | a |
-| test_free.cpp:343:12:343:24 | *access to array [post update] [ptr] | semmle.label | *access to array [post update] [ptr] |
-| test_free.cpp:343:26:343:28 | pointer to operator delete output argument | semmle.label | pointer to operator delete output argument |
-| test_free.cpp:344:12:344:24 | *access to array [post update] [ptr] | semmle.label | *access to array [post update] [ptr] |
-| test_free.cpp:344:12:344:24 | *access to array [ptr] | semmle.label | *access to array [ptr] |
-| test_free.cpp:344:26:344:28 | pointer to operator delete output argument | semmle.label | pointer to operator delete output argument |
-| test_free.cpp:344:26:344:28 | ptr | semmle.label | ptr |
-| test_free.cpp:345:12:345:24 | *access to array [post update] [ptr] | semmle.label | *access to array [post update] [ptr] |
-| test_free.cpp:345:12:345:24 | *access to array [ptr] | semmle.label | *access to array [ptr] |
-| test_free.cpp:345:12:345:24 | *access to array [ptr] | semmle.label | *access to array [ptr] |
-| test_free.cpp:345:26:345:28 | pointer to operator delete output argument | semmle.label | pointer to operator delete output argument |
-| test_free.cpp:345:26:345:28 | ptr | semmle.label | ptr |
-| test_free.cpp:345:26:345:28 | ptr | semmle.label | ptr |
-| test_free.cpp:346:12:346:24 | *access to array [ptr] | semmle.label | *access to array [ptr] |
-| test_free.cpp:346:12:346:24 | *access to array [ptr] | semmle.label | *access to array [ptr] |
-| test_free.cpp:346:12:346:24 | *access to array [ptr] | semmle.label | *access to array [ptr] |
-| test_free.cpp:346:26:346:28 | ptr | semmle.label | ptr |
-| test_free.cpp:346:26:346:28 | ptr | semmle.label | ptr |
-| test_free.cpp:346:26:346:28 | ptr | semmle.label | ptr |
 subpaths
 #select
 | test_free.cpp:14:10:14:10 | a | test_free.cpp:11:10:11:10 | pointer to free output argument | test_free.cpp:14:10:14:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:14:10:14:10 | a | a | test_free.cpp:11:5:11:8 | call to free | call to free |
@@ -91,9 +58,3 @@ subpaths
 | test_free.cpp:209:10:209:10 | a | test_free.cpp:207:10:207:10 | pointer to free output argument | test_free.cpp:209:10:209:10 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:209:10:209:10 | a | a | test_free.cpp:207:5:207:8 | call to free | call to free |
 | test_free.cpp:302:12:302:14 | buf | test_free.cpp:301:12:301:14 | pointer to g_free output argument | test_free.cpp:302:12:302:14 | buf | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:302:12:302:14 | buf | buf | test_free.cpp:301:5:301:10 | call to g_free | call to g_free |
 | test_free.cpp:322:12:322:12 | a | test_free.cpp:319:16:319:16 | pointer to operator delete output argument | test_free.cpp:322:12:322:12 | a | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:322:12:322:12 | a | a | test_free.cpp:319:9:319:16 | delete | delete |
-| test_free.cpp:344:26:344:28 | ptr | test_free.cpp:343:26:343:28 | pointer to operator delete output argument | test_free.cpp:344:26:344:28 | ptr | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:344:26:344:28 | ptr | ptr | test_free.cpp:343:5:343:28 | delete | delete |
-| test_free.cpp:345:26:345:28 | ptr | test_free.cpp:343:26:343:28 | pointer to operator delete output argument | test_free.cpp:345:26:345:28 | ptr | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:345:26:345:28 | ptr | ptr | test_free.cpp:343:5:343:28 | delete | delete |
-| test_free.cpp:345:26:345:28 | ptr | test_free.cpp:344:26:344:28 | pointer to operator delete output argument | test_free.cpp:345:26:345:28 | ptr | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:345:26:345:28 | ptr | ptr | test_free.cpp:344:5:344:28 | delete | delete |
-| test_free.cpp:346:26:346:28 | ptr | test_free.cpp:343:26:343:28 | pointer to operator delete output argument | test_free.cpp:346:26:346:28 | ptr | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:346:26:346:28 | ptr | ptr | test_free.cpp:343:5:343:28 | delete | delete |
-| test_free.cpp:346:26:346:28 | ptr | test_free.cpp:344:26:344:28 | pointer to operator delete output argument | test_free.cpp:346:26:346:28 | ptr | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:346:26:346:28 | ptr | ptr | test_free.cpp:344:5:344:28 | delete | delete |
-| test_free.cpp:346:26:346:28 | ptr | test_free.cpp:345:26:345:28 | pointer to operator delete output argument | test_free.cpp:346:26:346:28 | ptr | Memory pointed to by $@ may already have been freed by $@. | test_free.cpp:346:26:346:28 | ptr | ptr | test_free.cpp:345:5:345:28 | delete | delete |

cpp/ql/test/query-tests/Critical/MemoryFreed/MemoryFreed.expected

@@ -115,6 +115,8 @@
 | test_free.cpp:344:26:344:28 | ptr |
 | test_free.cpp:345:26:345:28 | ptr |
 | test_free.cpp:346:26:346:28 | ptr |
+| test_free.cpp:356:19:356:19 | a |
+| test_free.cpp:357:19:357:19 | a |
 | virtual.cpp:18:10:18:10 | a |
 | virtual.cpp:19:10:19:10 | c |
 | virtual.cpp:38:10:38:10 | b |

cpp/ql/test/query-tests/Critical/MemoryFreed/test_free.cpp

@@ -341,7 +341,18 @@ struct PtrContainer {
 
 void test_array(PtrContainer *containers) {
     delete containers[0].ptr; // GOOD
-    delete containers[1].ptr; // GOOD [FALSE POSITIVE]
+    delete containers[1].ptr; // GOOD
-    delete containers[2].ptr; // GOOD [FALSE POSITIVE]
+    delete containers[2].ptr; // GOOD
-    delete containers[2].ptr; // BAD (double free)
+    delete containers[2].ptr; // BAD (double free) [NOT DETECTED]
 }
+
+struct E {
+    struct EC {
+        int* a;
+    } ec[2];
+};
+
+void test(E* e) {
+    free(e->ec[0].a);
+    free(e->ec[1].a); // GOOD
+}

cpp/ql/test/query-tests/Security/CWE/CWE-570/IncorrectAllocationErrorHandling.expected

@@ -1,15 +1,15 @@
-| test.cpp:21:9:21:15 | new | This allocation cannot return null. $@ is unnecessary. | test.cpp:21:9:21:15 | new | This check |
+| test.cpp:21:9:21:15 | new | This allocation cannot return null. $@ is unnecessary. | test.cpp:21:7:21:16 | ! ... | This check |
 | test.cpp:29:13:29:24 | new[] | This allocation cannot return null. $@ is unnecessary. | test.cpp:30:7:30:13 | ... == ... | This check |
-| test.cpp:33:13:33:24 | new[] | This allocation cannot return null. $@ is unnecessary. | test.cpp:34:8:34:9 | p2 | This check |
+| test.cpp:33:13:33:24 | new[] | This allocation cannot return null. $@ is unnecessary. | test.cpp:34:7:34:9 | ! ... | This check |
 | test.cpp:37:13:37:24 | new[] | This allocation cannot return null. $@ is unnecessary. | test.cpp:38:7:38:16 | ... == ... | This check |
 | test.cpp:41:13:41:24 | new[] | This allocation cannot return null. $@ is unnecessary. | test.cpp:42:7:42:19 | ... == ... | This check |
 | test.cpp:45:13:45:24 | new[] | This allocation cannot return null. $@ is unnecessary. | test.cpp:46:7:46:8 | p5 | This check |
 | test.cpp:49:8:49:19 | new[] | This allocation cannot return null. $@ is unnecessary. | test.cpp:50:7:50:13 | ... == ... | This check |
-| test.cpp:53:8:53:19 | new[] | This allocation cannot return null. $@ is unnecessary. | test.cpp:54:8:54:9 | p7 | This check |
+| test.cpp:53:8:53:19 | new[] | This allocation cannot return null. $@ is unnecessary. | test.cpp:54:7:54:9 | ! ... | This check |
 | test.cpp:58:8:58:19 | new[] | This allocation cannot return null. $@ is unnecessary. | test.cpp:59:7:59:16 | ... == ... | This check |
 | test.cpp:63:8:63:19 | new[] | This allocation cannot return null. $@ is unnecessary. | test.cpp:64:7:64:19 | ... != ... | This check |
 | test.cpp:69:9:69:20 | new[] | This allocation cannot return null. $@ is unnecessary. | test.cpp:70:7:70:14 | ... != ... | This check |
-| test.cpp:75:11:75:22 | new[] | This allocation cannot return null. $@ is unnecessary. | test.cpp:76:13:76:15 | p11 | This check |
+| test.cpp:75:11:75:22 | new[] | This allocation cannot return null. $@ is unnecessary. | test.cpp:76:12:76:15 | ! ... | This check |
 | test.cpp:92:5:92:31 | new[] | This allocation cannot throw. $@ is unnecessary. | test.cpp:97:36:98:3 | { ... } | This catch block |
 | test.cpp:93:15:93:41 | new[] | This allocation cannot throw. $@ is unnecessary. | test.cpp:97:36:98:3 | { ... } | This catch block |
 | test.cpp:96:10:96:36 | new[] | This allocation cannot throw. $@ is unnecessary. | test.cpp:97:36:98:3 | { ... } | This catch block |

csharp/documentation/library-coverage/coverage.csv

@@ -2,43 +2,44 @@ package,sink,source,summary,sink:code-injection,sink:encryption-decryptor,sink:e
 Amazon.Lambda.APIGatewayEvents,,6,,,,,,,,,,,,,,,,,,6,,,
 Amazon.Lambda.Core,10,,,,,,,,,,,10,,,,,,,,,,,
 Dapper,55,42,1,,,,,,,,,,55,,42,,,,,,,,1
-ILCompiler,,,81,,,,,,,,,,,,,,,,,,,81,
+ILCompiler,,,123,,,,,,,,,,,,,,,,,,,123,
-ILLink.RoslynAnalyzer,,,63,,,,,,,,,,,,,,,,,,,63,
+ILLink.RoslynAnalyzer,,,145,,,,,,,,,,,,,,,,,,,145,
-ILLink.Shared,,,32,,,,,,,,,,,,,,,,,,,30,2
+ILLink.Shared,,,34,,,,,,,,,,,,,,,,,,,32,2
-ILLink.Tasks,,,3,,,,,,,,,,,,,,,,,,,3,
+ILLink.Tasks,,,4,,,,,,,,,,,,,,,,,,,4,
 Internal.IL,,,46,,,,,,,,,,,,,,,,,,,44,2
 Internal.Pgo,,,9,,,,,,,,,,,,,,,,,,,8,1
-Internal.TypeSystem,,,291,,,,,,,,,,,,,,,,,,,275,16
+Internal.TypeSystem,,,315,,,,,,,,,,,,,,,,,,,299,16
-JsonToItemsTaskFactory,,,5,,,,,,,,,,,,,,,,,,,5,
+JsonToItemsTaskFactory,,,10,,,,,,,,,,,,,,,,,,,10,
-Microsoft.Android.Build,,,14,,,,,,,,,,,,,,,,,,,14,
+Microsoft.Android.Build,,,16,,,,,,,,,,,,,,,,,,,16,
-Microsoft.Apple.Build,,,5,,,,,,,,,,,,,,,,,,,5,
+Microsoft.Apple.Build,,,8,,,,,,,,,,,,,,,,,,,8,
 Microsoft.ApplicationBlocks.Data,28,,,,,,,,,,,,28,,,,,,,,,,
-Microsoft.CSharp,,,10,,,,,,,,,,,,,,,,,,,10,
+Microsoft.CSharp,,,13,,,,,,,,,,,,,,,,,,,13,
 Microsoft.Diagnostics.Tools.Pgo,,,12,,,,,,,,,,,,,,,,,,,12,
+Microsoft.DotNet.Build.Tasks,,,6,,,,,,,,,,,,,,,,,,,6,
 Microsoft.EntityFrameworkCore,6,,12,,,,,,,,,,6,,,,,,,,,,12
-Microsoft.Extensions.Caching.Distributed,,,9,,,,,,,,,,,,,,,,,,,9,
+Microsoft.Extensions.Caching.Distributed,,,10,,,,,,,,,,,,,,,,,,,10,
-Microsoft.Extensions.Caching.Memory,,,30,,,,,,,,,,,,,,,,,,,29,1
+Microsoft.Extensions.Caching.Memory,,,39,,,,,,,,,,,,,,,,,,,38,1
-Microsoft.Extensions.Configuration,,2,77,,,,,,,,,,,,,2,,,,,,76,1
+Microsoft.Extensions.Configuration,,2,90,,,,,,,,,,,,,2,,,,,,89,1
-Microsoft.Extensions.DependencyInjection,,,96,,,,,,,,,,,,,,,,,,,95,1
+Microsoft.Extensions.DependencyInjection,,,134,,,,,,,,,,,,,,,,,,,133,1
-Microsoft.Extensions.DependencyModel,,,9,,,,,,,,,,,,,,,,,,,9,
+Microsoft.Extensions.DependencyModel,,,18,,,,,,,,,,,,,,,,,,,18,
 Microsoft.Extensions.Diagnostics.Metrics,,,15,,,,,,,,,,,,,,,,,,,15,
 Microsoft.Extensions.FileProviders,,,15,,,,,,,,,,,,,,,,,,,15,
-Microsoft.Extensions.FileSystemGlobbing,,,16,,,,,,,,,,,,,,,,,,,14,2
+Microsoft.Extensions.FileSystemGlobbing,,,18,,,,,,,,,,,,,,,,,,,16,2
-Microsoft.Extensions.Hosting,,,26,,,,,,,,,,,,,,,,,,,25,1
+Microsoft.Extensions.Hosting,,,41,,,,,,,,,,,,,,,,,,,40,1
-Microsoft.Extensions.Http,,,8,,,,,,,,,,,,,,,,,,,8,
+Microsoft.Extensions.Http,,,9,,,,,,,,,,,,,,,,,,,9,
-Microsoft.Extensions.Logging,,,53,,,,,,,,,,,,,,,,,,,52,1
+Microsoft.Extensions.Logging,,,65,,,,,,,,,,,,,,,,,,,64,1
-Microsoft.Extensions.Options,,,8,,,,,,,,,,,,,,,,,,,8,
+Microsoft.Extensions.Options,,,13,,,,,,,,,,,,,,,,,,,13,
-Microsoft.Extensions.Primitives,,,64,,,,,,,,,,,,,,,,,,,64,
+Microsoft.Extensions.Primitives,,,72,,,,,,,,,,,,,,,,,,,72,
-Microsoft.Interop,,,73,,,,,,,,,,,,,,,,,,,73,
+Microsoft.Interop,,,121,,,,,,,,,,,,,,,,,,,121,
-Microsoft.NET.Build.Tasks,,,1,,,,,,,,,,,,,,,,,,,1,
+Microsoft.NET.Build.Tasks,,,4,,,,,,,,,,,,,,,,,,,4,
-Microsoft.NET.WebAssembly.Webcil,,,7,,,,,,,,,,,,,,,,,,,7,
+Microsoft.NET.WebAssembly.Webcil,,,8,,,,,,,,,,,,,,,,,,,8,
 Microsoft.VisualBasic,,,6,,,,,,,,,,,,,,,,,,,1,5
-Microsoft.WebAssembly.Build.Tasks,,,3,,,,,,,,,,,,,,,,,,,3,
+Microsoft.WebAssembly.Build.Tasks,,,4,,,,,,,,,,,,,,,,,,,4,
 Microsoft.Win32,,4,4,,,,,,,,,,,,,,,,,,4,4,
-Mono.Linker,,,158,,,,,,,,,,,,,,,,,,,158,
+Mono.Linker,,,285,,,,,,,,,,,,,,,,,,,285,
 MySql.Data.MySqlClient,48,,,,,,,,,,,,48,,,,,,,,,,
 Newtonsoft.Json,,,91,,,,,,,,,,,,,,,,,,,73,18
 ServiceStack,194,,7,27,,,,,75,,,,92,,,,,,,,,7,
-SourceGenerators,,,4,,,,,,,,,,,,,,,,,,,4,
+SourceGenerators,,,5,,,,,,,,,,,,,,,,,,,5,
-System,49,44,9873,,3,3,1,,,4,5,,33,2,,3,15,17,3,4,,7968,1905
+System,60,44,10614,,7,6,5,,,4,5,,33,2,,3,15,17,3,4,,8709,1905
 Windows.Security.Cryptography.Core,1,,,,,,,1,,,,,,,,,,,,,,,

csharp/documentation/library-coverage/coverage.rst

@@ -8,7 +8,7 @@ C# framework & library support
 
    Framework / library,Package,Flow sources,Taint & value steps,Sinks (total),`CWE-079` :sub:`Cross-site scripting`
    `ServiceStack <https://servicestack.net/>`_,"``ServiceStack.*``, ``ServiceStack``",,7,194,
-   System,"``System.*``, ``System``",44,9873,49,9
+   System,"``System.*``, ``System``",44,10614,60,9
-   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",54,1357,148,
+   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.DotNet.Build.Tasks``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",54,1821,148,
-   Totals,,98,11237,391,9
+   Totals,,98,12442,402,9
 

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -42,17 +42,17 @@ namespace Semmle.Extraction.CSharp.Standalone
                     (compilation, options) => analyser.Initialize(output.FullName, extractionInput.CompilationInfos, compilation, options),
                     () =>
                     {
-                        foreach (var type in analyser.MissingNamespaces)
+                        foreach (var type in analyser.ExtractionContext!.MissingNamespaces)
                         {
                             progressMonitor.MissingNamespace(type);
                         }
 
-                        foreach (var type in analyser.MissingTypes)
+                        foreach (var type in analyser.ExtractionContext!.MissingTypes)
                         {
                             progressMonitor.MissingType(type);
                         }
 
-                        progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
+                        progressMonitor.MissingSummary(analyser.ExtractionContext!.MissingTypes.Count(), analyser.ExtractionContext!.MissingNamespaces.Count());
                     });
             }
             finally
@@ -69,29 +69,6 @@ namespace Semmle.Extraction.CSharp.Standalone
             }
         }
 
-        private static void ExtractStandalone(
-            ExtractionInput extractionInput,
-            IProgressMonitor pm,
-            ILogger logger,
-            CommonOptions options)
-        {
-            var stopwatch = new Stopwatch();
-            stopwatch.Start();
-
-            var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
-            var pathTransformer = new PathTransformer(canonicalPathCache);
-
-            using var analyser = new StandaloneAnalyser(pm, logger, false, pathTransformer);
-            try
-            {
-                AnalyseStandalone(analyser, extractionInput, options, pm, stopwatch);
-            }
-            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
-            {
-                analyser.Logger.Log(Severity.Error, "  Unhandled exception: {0}", ex);
-            }
-        }
-
         private class ExtractionProgress : IProgressMonitor
         {
             public ExtractionProgress(ILogger output)
@@ -141,8 +118,8 @@ namespace Semmle.Extraction.CSharp.Standalone
 
         public static ExitCode Run(Options options)
         {
-            var stopwatch = new Stopwatch();
+            var overallStopwatch = new Stopwatch();
-            stopwatch.Start();
+            overallStopwatch.Start();
 
             using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true);
             logger.Log(Severity.Info, "Extracting C# with build-mode set to 'none'");
@@ -158,12 +135,26 @@ namespace Semmle.Extraction.CSharp.Standalone
 
             logger.Log(Severity.Info, "");
             logger.Log(Severity.Info, "Extracting...");
-            ExtractStandalone(
+
-                new ExtractionInput(dependencyManager.AllSourceFiles, dependencyManager.ReferenceFiles, dependencyManager.CompilationInfos),
+            var analyzerStopwatch = new Stopwatch();
-                new ExtractionProgress(logger),
+            analyzerStopwatch.Start();
-                fileLogger,
+
-                options);
+            var canonicalPathCache = CanonicalPathCache.Create(fileLogger, 1000);
-            logger.Log(Severity.Info, $"Extraction completed in {stopwatch.Elapsed}");
+            var pathTransformer = new PathTransformer(canonicalPathCache);
+
+            var progressMonitor = new ExtractionProgress(logger);
+            using var analyser = new StandaloneAnalyser(progressMonitor, fileLogger, pathTransformer, canonicalPathCache, false);
+            try
+            {
+                var extractionInput = new ExtractionInput(dependencyManager.AllSourceFiles, dependencyManager.ReferenceFiles, dependencyManager.CompilationInfos);
+                AnalyseStandalone(analyser, extractionInput, options, progressMonitor, analyzerStopwatch);
+            }
+            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
+            {
+                fileLogger.Log(Severity.Error, "  Unhandled exception: {0}", ex);
+            }
+
+            logger.Log(Severity.Info, $"Extraction completed in {overallStopwatch.Elapsed}");
 
             return ExitCode.Ok;
         }

csharp/extractor/Semmle.Extraction.CSharp/BUILD.bazel

@@ -11,7 +11,6 @@ codeql_csharp_library(
         "Extractor/**/*.cs",
         "Kinds/**/*.cs",
         "Populators/**/*.cs",
-        "Properties/**/*.cs",
         "*.cs",
     ]),
     allow_unsafe_blocks = True,

csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs

@@ -17,7 +17,7 @@ namespace Semmle.Extraction.CSharp.Entities
             isOutputAssembly = init is null;
             if (isOutputAssembly)
             {
-                assemblyPath = cx.Extractor.OutputPath;
+                assemblyPath = cx.ExtractionContext.OutputPath;
                 assembly = cx.Compilation.Assembly;
             }
             else
@@ -25,7 +25,7 @@ namespace Semmle.Extraction.CSharp.Entities
                 assembly = init!.MetadataModule!.ContainingAssembly;
                 var identity = assembly.Identity;
                 var idString = identity.Name + " " + identity.Version;
-                assemblyPath = cx.Extractor.GetAssemblyFile(idString);
+                assemblyPath = cx.ExtractionContext.GetAssemblyFile(idString);
             }
         }
 
@@ -33,7 +33,7 @@ namespace Semmle.Extraction.CSharp.Entities
         {
             if (assemblyPath is not null)
             {
-                var isBuildlessOutputAssembly = isOutputAssembly && Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone);
+                var isBuildlessOutputAssembly = isOutputAssembly && Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone);
                 var identifier = isBuildlessOutputAssembly
                     ? ""
                     : assembly.ToString() ?? "";
@@ -74,7 +74,7 @@ namespace Semmle.Extraction.CSharp.Entities
 
         public override void WriteId(EscapingTextWriter trapFile)
         {
-            if (isOutputAssembly && Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
+            if (isOutputAssembly && Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone))
             {
                 trapFile.Write("buildlessOutputAssembly");
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs

@@ -18,8 +18,8 @@ namespace Semmle.Extraction.CSharp.Entities
 #nullable disable warnings
         private Compilation(Context cx) : base(cx, null)
         {
-            cwd = cx.Extractor.Cwd;
+            cwd = cx.ExtractionContext.Cwd;
-            args = cx.Extractor.Args;
+            args = cx.ExtractionContext.Args;
             hashCode = cwd.GetHashCode();
             for (var i = 0; i < args.Length; i++)
             {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs

@@ -28,7 +28,7 @@ namespace Semmle.Extraction.CSharp.Entities
             {
                 if (messageCount == limit + 1)
                 {
-                    Context.Extractor.Logger.LogWarning($"Stopped logging {key} compiler diagnostics for the current compilation after reaching {limit}");
+                    Context.ExtractionContext.Logger.LogWarning($"Stopped logging {key} compiler diagnostics for the current compilation after reaching {limit}");
                 }
 
                 return;

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs

@@ -133,7 +133,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                         .Where(method => method.Parameters.Length >= Syntax.ArgumentList.Arguments.Count)
                         .Where(method => method.Parameters.Count(p => !p.HasExplicitDefaultValue) <= Syntax.ArgumentList.Arguments.Count);
 
-                    return Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) ?
+                    return Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone) ?
                         candidates.FirstOrDefault() :
                         candidates.SingleOrDefault();
                 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs

@@ -61,7 +61,7 @@ namespace Semmle.Extraction.CSharp.Entities
                 }
             }
 
-            trapFile.file_extraction_mode(this, Context.Extractor.Mode);
+            trapFile.file_extraction_mode(this, Context.ExtractionContext.Mode);
         }
 
         private bool IsPossiblyTextFile()

csharp/extractor/Semmle.Extraction.CSharp/Entities/NonGeneratedSourceLocation.cs

@@ -27,7 +27,7 @@ namespace Semmle.Extraction.CSharp.Entities
             var mapped = Symbol.GetMappedLineSpan();
             if (mapped.HasMappedPath && mapped.IsValid)
             {
-                var path = TryAdjustRelativeMappedFilePath(mapped.Path, Position.Path, Context.Extractor.Logger);
+                var path = Context.TryAdjustRelativeMappedFilePath(mapped.Path, Position.Path);
                 var mappedLoc = Create(Context, Location.Create(path, default, mapped.Span));
 
                 trapFile.locations_mapped(this, mappedLoc);
@@ -64,25 +64,5 @@ namespace Semmle.Extraction.CSharp.Entities
 
             public override NonGeneratedSourceLocation Create(Context cx, Location init) => new NonGeneratedSourceLocation(cx, init);
         }
-
-        public static string TryAdjustRelativeMappedFilePath(string mappedToPath, string mappedFromPath, ILogger logger)
-        {
-            if (!Path.IsPathRooted(mappedToPath))
-            {
-                try
-                {
-                    var fullPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(mappedFromPath)!, mappedToPath));
-                    logger.LogDebug($"Found relative path in line mapping: '{mappedToPath}', interpreting it as '{fullPath}'");
-
-                    mappedToPath = fullPath;
-                }
-                catch (Exception e)
-                {
-                    logger.LogDebug($"Failed to compute absolute path for relative path in line mapping: '{mappedToPath}': {e}");
-                }
-            }
-
-            return mappedToPath;
-        }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs

@@ -63,7 +63,7 @@ namespace Semmle.Extraction.CSharp.Entities
         {
             if (method.MethodKind == MethodKind.ReducedExtension)
             {
-                cx.Extractor.Logger.Log(Semmle.Util.Logging.Severity.Warning, "Reduced extension method symbols should not be directly extracted.");
+                cx.ExtractionContext.Logger.Log(Semmle.Util.Logging.Severity.Warning, "Reduced extension method symbols should not be directly extracted.");
             }
 
             return OrdinaryMethodFactory.Instance.CreateEntityFromSymbol(cx, method);

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/LineOrSpanDirective.cs

@@ -28,7 +28,7 @@ namespace Semmle.Extraction.CSharp.Entities
             var path = Symbol.File.ValueText;
             if (!string.IsNullOrWhiteSpace(path))
             {
-                path = NonGeneratedSourceLocation.TryAdjustRelativeMappedFilePath(path, Symbol.SyntaxTree.FilePath, Context.Extractor.Logger);
+                path = Context.TryAdjustRelativeMappedFilePath(path, Symbol.SyntaxTree.FilePath);
                 var file = File.Create(Context, path);
                 trapFile.directive_line_file(this, file);
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PragmaChecksumDirective.cs

@@ -12,7 +12,7 @@ namespace Semmle.Extraction.CSharp.Entities
 
         protected override void PopulatePreprocessor(TextWriter trapFile)
         {
-            var path = NonGeneratedSourceLocation.TryAdjustRelativeMappedFilePath(Symbol.File.ValueText, Symbol.SyntaxTree.FilePath, Context.Extractor.Logger);
+            var path = Context.TryAdjustRelativeMappedFilePath(Symbol.File.ValueText, Symbol.SyntaxTree.FilePath);
             var file = File.Create(Context, path);
             trapFile.pragma_checksums(this, file, Symbol.Guid.ToString(), Symbol.Bytes.ToString());
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs

@@ -36,7 +36,7 @@ namespace Semmle.Extraction.CSharp.Entities
             if (Symbol.TypeKind == TypeKind.Error)
             {
                 UnknownType.Create(Context); // make sure this exists so we can use it in `TypeRef::getReferencedType`
-                Context.Extractor.MissingType(Symbol.ToString()!, Context.FromSource);
+                Context.ExtractionContext.MissingType(Symbol.ToString()!, Context.FromSource);
                 return;
             }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs

@@ -36,7 +36,7 @@ namespace Semmle.Extraction.CSharp.Entities
                 }
                 else
                 {
-                    Context.Extractor.MissingNamespace(name.ToFullString(), Context.FromSource);
+                    Context.ExtractionContext.MissingNamespace(name.ToFullString(), Context.FromSource);
                     Context.ModelError(node, "Namespace not found");
                     return;
                 }

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -10,6 +10,7 @@ using Microsoft.CodeAnalysis.CSharp;
 using Semmle.Util;
 using Semmle.Util.Logging;
 using Semmle.Extraction.CSharp.Populators;
+using System.Reflection;
 
 namespace Semmle.Extraction.CSharp
 {
@@ -18,7 +19,7 @@ namespace Semmle.Extraction.CSharp
     /// </summary>
     public class Analyser : IDisposable
     {
-        protected Extraction.Extractor? extractor;
+        public ExtractionContext? ExtractionContext { get; protected set; }
         protected CSharpCompilation? compilation;
         protected CommonOptions? options;
         private protected Entities.Compilation? compilationEntity;
@@ -38,14 +39,23 @@ namespace Semmle.Extraction.CSharp
 
         public PathTransformer PathTransformer { get; }
 
-        protected Analyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer)
+        public IPathCache PathCache { get; }
+
+        protected Analyser(
+            IProgressMonitor pm,
+            ILogger logger,
+            PathTransformer pathTransformer,
+            IPathCache pathCache,
+            bool addAssemblyTrapPrefix)
         {
             Logger = logger;
+            PathTransformer = pathTransformer;
+            PathCache = pathCache;
             this.addAssemblyTrapPrefix = addAssemblyTrapPrefix;
+            this.progressMonitor = pm;
+
             Logger.Log(Severity.Info, "EXTRACTION STARTING at {0}", DateTime.Now);
             stopWatch.Start();
-            progressMonitor = pm;
-            PathTransformer = pathTransformer;
         }
 
         /// <summary>
@@ -98,12 +108,12 @@ namespace Semmle.Extraction.CSharp
                         var def = reader.GetAssemblyDefinition();
                         assemblyIdentity = reader.GetString(def.Name) + " " + def.Version;
                     }
-                    extractor.SetAssemblyFile(assemblyIdentity, refPath);
+                    ExtractionContext.SetAssemblyFile(assemblyIdentity, refPath);
 
                 }
                 catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
                 {
-                    extractor.Message(new Message("Exception reading reference file", reference.FilePath, null, ex.StackTrace));
+                    ExtractionContext.Message(new Message("Exception reading reference file", reference.FilePath, null, ex.StackTrace));
                 }
             }
         }
@@ -148,7 +158,7 @@ namespace Semmle.Extraction.CSharp
 
                     if (compilation.GetAssemblyOrModuleSymbol(r) is IAssemblySymbol assembly)
                     {
-                        var cx = new Context(extractor, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
+                        var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
 
                         foreach (var module in assembly.Modules)
                         {
@@ -191,7 +201,7 @@ namespace Semmle.Extraction.CSharp
 
                 if (!upToDate)
                 {
-                    var cx = new Context(extractor, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
+                    var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
                     // Ensure that the file itself is populated in case the source file is totally empty
                     var root = tree.GetRoot();
                     Entities.File.Create(cx, root.SyntaxTree.FilePath);
@@ -213,7 +223,7 @@ namespace Semmle.Extraction.CSharp
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
-                extractor.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", tree.FilePath, null, ex.StackTrace));
+                ExtractionContext.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", tree.FilePath, null, ex.StackTrace));
             }
         }
 
@@ -221,7 +231,7 @@ namespace Semmle.Extraction.CSharp
         {
             try
             {
-                var assemblyPath = extractor.OutputPath;
+                var assemblyPath = ExtractionContext.OutputPath;
                 var stopwatch = new Stopwatch();
                 stopwatch.Start();
                 var currentTaskId = IncrementTaskCount();
@@ -231,11 +241,11 @@ namespace Semmle.Extraction.CSharp
                 var assembly = compilation.Assembly;
                 var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
                 compilationTrapFile = trapWriter;  // Dispose later
-                var cx = new Context(extractor, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
+                var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
 
                 compilationEntity = Entities.Compilation.Create(cx);
 
-                extractor.CompilationInfos.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value));
+                ExtractionContext.CompilationInfos.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value));
 
                 ReportProgressTaskDone(currentTaskId, assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, AnalysisAction.Extracted);
             }
@@ -318,7 +328,7 @@ namespace Semmle.Extraction.CSharp
         /// <summary>
         /// Number of errors encountered during extraction.
         /// </summary>
-        private int ExtractorErrors => extractor?.Errors ?? 0;
+        private int ExtractorErrors => ExtractionContext?.Errors ?? 0;
 
         /// <summary>
         /// Number of errors encountered by the compiler.
@@ -333,11 +343,26 @@ namespace Semmle.Extraction.CSharp
         /// <summary>
         /// Logs information about the extractor.
         /// </summary>
-        public void LogExtractorInfo(string extractorVersion)
+        public void LogExtractorInfo()
         {
             Logger.Log(Severity.Info, "  Extractor: {0}", Environment.GetCommandLineArgs().First());
-            Logger.Log(Severity.Info, "  Extractor version: {0}", extractorVersion);
+            Logger.Log(Severity.Info, "  Extractor version: {0}", Version);
             Logger.Log(Severity.Info, "  Current working directory: {0}", Directory.GetCurrentDirectory());
         }
+
+        private static string Version
+        {
+            get
+            {
+                // the attribute for the git information are always attached to the entry assembly by our build system
+                var assembly = Assembly.GetEntryAssembly();
+                var versionString = assembly?.GetCustomAttribute<AssemblyInformationalVersionAttribute>();
+                if (versionString == null)
+                {
+                    return "unknown (not built from internal bazel workspace)";
+                }
+                return versionString.InformationalVersion;
+            }
+        }
     }
 }