Add deprecation notice on swift+linux

.bazelrc

@@ -24,6 +24,5 @@ common --registry=file:///%workspace%/misc/bazel/registry
 common --registry=https://bcr.bazel.build
 
 common --@rules_dotnet//dotnet/settings:strict_deps=false
-common --experimental_isolated_extension_usages
 
 try-import %workspace%/local.bazelrc

.bazelrc.internal

@@ -8,4 +8,3 @@ common --registry=https://bcr.bazel.build
 # its implementation packages without providing any code itself.
 # We either can depend on internal implementation details, or turn of strict deps.
 common --@rules_dotnet//dotnet/settings:strict_deps=false
-common --experimental_isolated_extension_usages

.github/workflows/buildifier.yml

@@ -24,5 +24,5 @@ jobs:
           extra_args: >
             buildifier --all-files 2>&1 ||
             (
-              echo -e "In order to format all bazel files, please run:\n  bazel run //misc/bazel/buildifier"; exit 1
+              echo -e "In order to format all bazel files, please run:\n  bazel run //misc/bazel:buildifier"; exit 1
             )

.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
         name: Format bazel files
         files: \.(bazel|bzl)
         language: system
-        entry: bazel run //misc/bazel/buildifier
+        entry: bazel run //misc/bazel:buildifier
         pass_filenames: false
 
 #      DISABLED: can be enabled by copying this config and installing `pre-commit` with `--config` on the copy

MODULE.bazel

@@ -1,7 +1,6 @@
 module(
-    name = "ql",
+    name = "codeql",
     version = "0.0",
-    repo_name = "codeql",
 )
 
 # this points to our internal repository when `codeql` is checked out as a submodule thereof
@@ -15,7 +14,7 @@ local_path_override(
 # see https://registry.bazel.build/ for a list of available packages
 
 bazel_dep(name = "platforms", version = "0.0.10")
-bazel_dep(name = "rules_go", version = "0.49.0")
+bazel_dep(name = "rules_go", version = "0.48.0")
 bazel_dep(name = "rules_pkg", version = "0.10.1")
 bazel_dep(name = "rules_nodejs", version = "6.2.0-codeql.1")
 bazel_dep(name = "rules_python", version = "0.32.2")
@@ -24,20 +23,18 @@ bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl")
 bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
 bazel_dep(name = "fmt", version = "10.0.0")
 bazel_dep(name = "rules_kotlin", version = "1.9.4-codeql.1")
-bazel_dep(name = "gazelle", version = "0.38.0")
+bazel_dep(name = "gazelle", version = "0.37.0")
 bazel_dep(name = "rules_dotnet", version = "0.15.1")
 bazel_dep(name = "googletest", version = "1.14.0.bcr.1")
-bazel_dep(name = "rules_rust", version = "0.49.1")
+bazel_dep(name = "rules_rust", version = "0.46.0")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
-# crate_py but shortened due to Windows file path considerations
-cp = use_extension(
+crate = use_extension(
     "@rules_rust//crate_universe:extension.bzl",
     "crate",
-    isolate = True,
 )
-cp.from_cargo(
+crate.from_cargo(
     name = "py_deps",
     cargo_lockfile = "//python/extractor/tsg-python:Cargo.lock",
     manifests = [
@@ -45,23 +42,15 @@ cp.from_cargo(
         "//python/extractor/tsg-python/tsp:Cargo.toml",
     ],
 )
-use_repo(cp, "py_deps")
-
-# crate_ruby, but shortened due to windows file paths
-r = use_extension(
-    "@rules_rust//crate_universe:extension.bzl",
-    "crate",
-    isolate = True,
-)
-r.from_cargo(
-    name = "rd",
+crate.from_cargo(
+    name = "ruby_deps",
     cargo_lockfile = "//ruby/extractor:Cargo.lock",
     manifests = [
         "//ruby/extractor:Cargo.toml",
         "//ruby/extractor/codeql-extractor-fake-crate:Cargo.toml",
     ],
 )
-use_repo(r, ruby_deps = "rd")
+use_repo(crate, "py_deps", "ruby_deps")
 
 dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet")
 dotnet.toolchain(dotnet_version = "8.0.101")

cpp/downgrades/68930f3b81bbe3fdbb91c850deca1fec8072d62a/upgrade.properties

@@ -1,3 +0,0 @@
-description: description: Support explicit(bool) specifiers
-compatibility: full
-explicit_specifier_exprs.rel: delete

cpp/downgrades/e197626a6ebccd052d5c891975fccf8aebcc9803/upgrade.properties

@@ -1,3 +0,0 @@
-description: Add relation between deduction guides and class templates
-compatibility: full
-deduction_guide_for_class.rel: delete

cpp/ql/lib/CHANGELOG.md

@@ -1,18 +1,3 @@
-## 1.4.0
-
-### New Features
-
-* A `getTemplateClass` predicate was added to the `DeductionGuide` class to get the class template for which the deduction guide is a guide.
-* An `isExplicit` predicate was added to the `Function` class that determines whether the function was declared as explicit.
-* A `getExplicitExpr` predicate was added to the `Function` class that yields the constant boolean expression (if any) that conditionally determines whether the function is explicit.
-* A `isDestroyingDeleteDeallocation` predicate was added to the `NewOrNewArrayExpr` and `DeleteOrDeleteArrayExpr` classes to indicate whether the deallocation function is a destroying delete. 
-
-### Minor Analysis Improvements
-
-* The controlling expression of a `constexpr if` is now always recognized as an unevaluated expression.
-* Improved performance of alias analysis of large function bodies. In rare cases, alerts that depend on alias analysis of large function bodies may be affected.
-* A `UsingEnumDeclarationEntry` class has been added for C++ `using enum` declarations. As part of this, synthesized `UsingDeclarationEntry`s are no longer emitted for individual enumerators of the referenced enumeration.
-
 ## 1.3.0
 
 ### New Features

cpp/ql/lib/change-notes/2024-07-23-using-enum-declaration copy.md

@@ -0,0 +1,4 @@
+---
+category: feature
+---
+* A `isDestroyingDeleteDeallocation` predicate was added to the `NewOrNewArrayExpr` and `DeleteOrDeleteArrayExpr` classes to indicate whether the deallocation function is a destroying delete. 

cpp/ql/lib/change-notes/2024-07-23-using-enum-declaration.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* A `UsingEnumDeclarationEntry` class has been added for C++ `using enum` declarations. As part of this, synthesized `UsingDeclarationEntry`s are no longer emitted for individual enumerators of the referenced enumeration.

cpp/ql/lib/change-notes/2024-07-25-alias-analysis-perf.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Improved performance of alias analysis of large function bodies. In rare cases, alerts that depend on alias analysis of large function bodies may be affected.

cpp/ql/lib/change-notes/2024-07-31-constexpr-if.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The controlling expression of a `constexpr if` is now always recognized as an unevaluated expression.

cpp/ql/lib/change-notes/released/1.4.0.md

@@ -1,14 +0,0 @@
-## 1.4.0
-
-### New Features
-
-* A `getTemplateClass` predicate was added to the `DeductionGuide` class to get the class template for which the deduction guide is a guide.
-* An `isExplicit` predicate was added to the `Function` class that determines whether the function was declared as explicit.
-* A `getExplicitExpr` predicate was added to the `Function` class that yields the constant boolean expression (if any) that conditionally determines whether the function is explicit.
-* A `isDestroyingDeleteDeallocation` predicate was added to the `NewOrNewArrayExpr` and `DeleteOrDeleteArrayExpr` classes to indicate whether the deallocation function is a destroying delete. 
-
-### Minor Analysis Improvements
-
-* The controlling expression of a `constexpr if` is now always recognized as an unevaluated expression.
-* Improved performance of alias analysis of large function bodies. In rare cases, alerts that depend on alias analysis of large function bodies may be affected.
-* A `UsingEnumDeclarationEntry` class has been added for C++ `using enum` declarations. As part of this, synthesized `UsingDeclarationEntry`s are no longer emitted for individual enumerators of the referenced enumeration.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.4.0
+lastReleaseVersion: 1.3.0

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 1.4.1-dev
+version: 1.3.1-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/Function.qll

@@ -158,26 +158,6 @@ class Function extends Declaration, ControlFlowNode, AccessHolder, @function {
    */
   predicate isConsteval() { this.hasSpecifier("is_consteval") }
 
-  /**
-   * Holds if this function is declared to be `explicit`.
-   */
-  predicate isExplicit() { this.hasSpecifier("explicit") }
-
-  /**
-   * Gets the constant expression that determines whether the function is explicit.
-   *
-   * For example, for the following code the result is the expression `sizeof(T) == 1`:
-   * ```
-   * template<typename T> struct C {
-   *   explicit(sizeof(T) == 1)
-   *   C(const T);
-   * };
-   * ```
-   */
-  Expr getExplicitExpr() {
-    explicit_specifier_exprs(underlyingElement(this), unresolveElement(result))
-  }
-
   /**
    * Holds if this function is declared with `__attribute__((naked))` or
    * `__declspec(naked)`.
@@ -918,11 +898,4 @@ class UserDefinedLiteral extends Function {
  */
 class DeductionGuide extends Function {
   DeductionGuide() { functions(underlyingElement(this), _, 8) }
-
-  /**
-   * Gets the class template for which this is a deduction guide.
-   */
-  TemplateClass getTemplateClass() {
-    deduction_guide_for_class(underlyingElement(this), unresolveElement(result))
-  }
 }

cpp/ql/lib/semmle/code/cpp/dataflow/ExternalFlow.qll

@@ -953,3 +953,21 @@ private class SummarizedCallableAdapter extends SummarizedCallable {
     interpretSummary(this, _, _, _, provenance, _)
   }
 }
+
+// adapter class for converting Mad neutrals to `NeutralCallable`s
+private class NeutralCallableAdapter extends NeutralCallable {
+  string kind;
+  string provenance_;
+
+  NeutralCallableAdapter() {
+    // Neutral models have not been implemented for CPP.
+    none() and
+    exists(this) and
+    exists(kind) and
+    exists(provenance_)
+  }
+
+  override string getKind() { result = kind }
+
+  override predicate hasProvenance(Provenance provenance) { provenance = provenance_ }
+}

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl1.qll

@@ -290,6 +290,14 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll

@@ -290,6 +290,14 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll

@@ -290,6 +290,14 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl4.qll

@@ -290,6 +290,14 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplLocal.qll

@@ -290,6 +290,14 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl1.qll

@@ -290,6 +290,14 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll

@@ -290,6 +290,14 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll

@@ -290,6 +290,14 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll

@@ -290,6 +290,14 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmlecode.cpp.dbscheme

@@ -409,11 +409,6 @@ function_defaulted(unique int id: @function ref);
 
 function_prototyped(unique int id: @function ref)
 
-deduction_guide_for_class(
-    int id: @function ref,
-    int class_template: @usertype ref
-)
-
 member_function_this_type(
     unique int id: @function ref,
     int this_type: @type ref
@@ -921,11 +916,6 @@ varspecifiers(
     int spec_id: @specifier ref
 );
 
-explicit_specifier_exprs(
-    unique int func_id: @function ref,
-    int constant: @expr ref
-)
-
 attributes(
     unique int id: @attribute,
     int kind: int ref,

cpp/ql/lib/upgrades/9629fc87dab7dbed0771bf5ce22bce4d7f943b52/upgrade.properties

@@ -1,2 +0,0 @@
-description: Add relation between deduction guides and class templates
-compatibility: partial

cpp/ql/lib/upgrades/e197626a6ebccd052d5c891975fccf8aebcc9803/upgrade.properties

@@ -1,2 +0,0 @@
-description: Support explicit(bool) specifiers
-compatibility: partial

cpp/ql/src/CHANGELOG.md

@@ -1,16 +1,3 @@
-## 1.2.0
-
-### Query Metadata Changes
-
-* The precision of `cpp/unsigned-difference-expression-compared-zero` ("Unsigned difference expression compared to zero") has been increased to `high`. As a result, it will be run by default as part of the Code Scanning suite.
-
-### Minor Analysis Improvements
-
-* Fixed false positives in the `cpp/memory-may-not-be-freed` ("Memory may not be freed") query involving class methods that returned an allocated field of that class being misidentified as allocators.
-* The `cpp/incorrectly-checked-scanf` ("Incorrect return-value check for a 'scanf'-like function") query now produces fewer false positive results.
-* The `cpp/incorrect-allocation-error-handling` ("Incorrect allocation-error handling") query no longer produces occasional false positive results inside template instantiations.
-* The `cpp/suspicious-allocation-size` ("Not enough memory allocated for array of pointer type") query no longer produces false positives on "variable size" `struct`s.
-
 ## 1.1.0
 
 ### Query Metadata Changes

cpp/ql/src/Security/CWE/CWE-190/Bounded.qll

@@ -24,10 +24,6 @@ private predicate boundedBitwiseAnd(Expr e, Expr andExpr, Expr operand1, Expr op
  * operation that may greatly reduce the range of possible values.
  */
 predicate bounded(Expr e) {
-  // There can be two separate reasons for `convertedExprMightOverflow` not holding:
-  // 1. `e` really cannot overflow.
-  // 2. `e` isn't analyzable.
-  // If we didn't rule out case 2 we would declare anything that isn't analyzable as bounded.
   (
     e instanceof UnaryArithmeticOperation or
     e instanceof BinaryArithmeticOperation or

cpp/ql/src/Security/CWE/CWE-190/TaintedAllocationSize.c

@@ -1,9 +1,11 @@
 int factor = atoi(getenv("BRANCHING_FACTOR"));
 
-// BAD: This can allocate too little memory if factor is very large due to overflow.
-char **root_node = (char **) malloc(factor * sizeof(char *));
-
-// GOOD: Prevent overflow and unbounded allocation size by checking the input.
-if (factor > 0 && factor <= 1000) {
-    char **root_node = (char **) malloc(factor * sizeof(char *));
+// GOOD: Prevent overflow by checking the input
+if (factor < 0 || factor > 1000) {
+    log("Factor out of range (%d)\n", factor);
+    return -1;
 }
+
+// This line can allocate too little memory if factor
+// is very large.
+char **root_node = (char **) malloc(factor * sizeof(char *));

cpp/ql/src/Security/CWE/CWE-190/TaintedAllocationSize.qhelp

@@ -3,16 +3,12 @@
   "qhelp.dtd">
 <qhelp>
 <overview>
-
-<p>This code allocates memory using a size value based on user input,
-with no apparent bound on its magnitude being established. This allows
-for arbitrary amounts of memory to be allocated.</p>
-
-<p>If the allocation size is calculated by multiplying user input by a
-<code>sizeof</code> expression, the multiplication can overflow. When
-an integer multiplication overflows in C, the result wraps around and
-can be much smaller than intended. A later attempt to write data into
-the allocated memory can then be out of bounds.</p>
+<p>This code calculates an allocation size by multiplying a user input
+by a <code>sizeof</code> expression. Since the user input has no
+apparent guard on its magnitude, this multiplication can
+overflow. When an integer multiply overflows in C, the result can wrap
+around and be much smaller than intended. A later attempt to put data
+into the allocated buffer can then overflow.</p>
 
 </overview>
 <recommendation>

cpp/ql/src/Security/CWE/CWE-190/TaintedAllocationSize.ql

@@ -1,7 +1,7 @@
 /**
- * @name Uncontrolled allocation size
- * @description Allocating memory with a size controlled by an external user can result in
- *              arbitrary amounts of memory being allocated.
+ * @name Overflow in uncontrolled allocation size
+ * @description Allocating memory with a size controlled by an external
+ *              user can result in integer overflow.
  * @kind path-problem
  * @problem.severity error
  * @security-severity 8.1
@@ -20,7 +20,6 @@ import semmle.code.cpp.ir.IR
 import semmle.code.cpp.controlflow.IRGuards
 import semmle.code.cpp.security.FlowSources
 import TaintedAllocationSize::PathGraph
-import Bounded
 
 /**
  * Holds if `alloc` is an allocation, and `tainted` is a child of it that is a
@@ -62,7 +61,16 @@ module TaintedAllocationSizeConfig implements DataFlow::ConfigSig {
 
   predicate isBarrier(DataFlow::Node node) {
     exists(Expr e | e = node.asExpr() |
-      bounded(e)
+      // There can be two separate reasons for `convertedExprMightOverflow` not holding:
+      // 1. `e` really cannot overflow.
+      // 2. `e` isn't analyzable.
+      // If we didn't rule out case 2 we would place barriers on anything that isn't analyzable.
+      (
+        e instanceof UnaryArithmeticOperation or
+        e instanceof BinaryArithmeticOperation or
+        e instanceof AssignArithmeticOperation
+      ) and
+      not convertedExprMightOverflow(e)
       or
       // Subtracting two pointers is either well-defined (and the result will likely be small), or
       // terribly undefined and dangerous. Here, we assume that the programmer has ensured that the
@@ -96,6 +104,5 @@ where
   isFlowSource(source.getNode(), taintCause) and
   TaintedAllocationSize::flowPath(source, sink) and
   allocSink(alloc, sink.getNode())
-select alloc, source, sink,
-  "This allocation size is derived from $@ and could allocate arbitrary amounts of memory.",
+select alloc, source, sink, "This allocation size is derived from $@ and might overflow.",
   source.getNode(), "user input (" + taintCause + ")"

cpp/ql/src/change-notes/2024-07-11-unsigned-difference-expression-compared-zero-query.md

@@ -0,0 +1,4 @@
+---
+category: queryMetadata
+---
+* The precision of `cpp/unsigned-difference-expression-compared-zero` ("Unsigned difference expression compared to zero") has been increased to `high`. As a result, it will be run by default as part of the Code Scanning suite.

cpp/ql/src/change-notes/2024-07-22-incorrect-allocation-error-handling.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The `cpp/incorrect-allocation-error-handling` ("Incorrect allocation-error handling") query no longer produces occasional false positive results inside template instantiations.

cpp/ql/src/change-notes/2024-07-22-suspicious-allocation-size.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The `cpp/suspicious-allocation-size` ("Not enough memory allocated for array of pointer type") query no longer produces false positives on "variable size" `struct`s.

cpp/ql/src/change-notes/2024-07-23-incorrectly-checked-scanf.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The `cpp/incorrectly-checked-scanf` ("Incorrect return-value check for a 'scanf'-like function") query now produces fewer false positive results.

cpp/ql/src/change-notes/2024-07-31-memory-may-not-be-freed.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Fixed false positives in the `cpp/memory-may-not-be-freed` ("Memory may not be freed") query involving class methods that returned an allocated field of that class being misidentified as allocators.

cpp/ql/src/change-notes/2024-08-16-uncontrolled-allocation-size.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The `cpp/uncontrolled-allocation-size` ("Uncontrolled allocation size") query now considers arithmetic operations that might reduce the size of user input as a barrier. The query therefore produces fewer false positive results.

cpp/ql/src/change-notes/released/1.2.0.md

@@ -1,12 +0,0 @@
-## 1.2.0
-
-### Query Metadata Changes
-
-* The precision of `cpp/unsigned-difference-expression-compared-zero` ("Unsigned difference expression compared to zero") has been increased to `high`. As a result, it will be run by default as part of the Code Scanning suite.
-
-### Minor Analysis Improvements
-
-* Fixed false positives in the `cpp/memory-may-not-be-freed` ("Memory may not be freed") query involving class methods that returned an allocated field of that class being misidentified as allocators.
-* The `cpp/incorrectly-checked-scanf` ("Incorrect return-value check for a 'scanf'-like function") query now produces fewer false positive results.
-* The `cpp/incorrect-allocation-error-handling` ("Incorrect allocation-error handling") query no longer produces occasional false positive results inside template instantiations.
-* The `cpp/suspicious-allocation-size` ("Not enough memory allocated for array of pointer type") query no longer produces false positives on "variable size" `struct`s.

cpp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.2.0
+lastReleaseVersion: 1.1.0

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 1.2.1-dev
+version: 1.1.1-dev
 groups:
   - cpp
   - queries

cpp/ql/test/library-tests/dataflow/external-models/flow.expected

@@ -11,14 +11,14 @@ edges
 | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:103:29:103:39 | *send_buffer | provenance |  Sink:MaD:6 |
 | asio_streams.cpp:100:64:100:71 | *send_str | asio_streams.cpp:56:18:56:23 | [summary param] *0 in buffer | provenance |  |
 | asio_streams.cpp:100:64:100:71 | *send_str | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | MaD:10 |
-| test.cpp:4:5:4:11 | [summary param] 0 in ymlStep | test.cpp:4:5:4:11 | [summary] to write: ReturnValue in ymlStep | provenance | MaD:644 |
-| test.cpp:7:10:7:18 | call to ymlSource | test.cpp:7:10:7:18 | call to ymlSource | provenance | Src:MaD:642  |
-| test.cpp:7:10:7:18 | call to ymlSource | test.cpp:11:10:11:10 | x | provenance |  Sink:MaD:643 |
+| test.cpp:4:5:4:11 | [summary param] 0 in ymlStep | test.cpp:4:5:4:11 | [summary] to write: ReturnValue in ymlStep | provenance | MaD:2 |
+| test.cpp:7:10:7:18 | call to ymlSource | test.cpp:7:10:7:18 | call to ymlSource | provenance | Src:MaD:0  |
+| test.cpp:7:10:7:18 | call to ymlSource | test.cpp:11:10:11:10 | x | provenance |  Sink:MaD:1 |
 | test.cpp:7:10:7:18 | call to ymlSource | test.cpp:13:18:13:18 | x | provenance |  |
 | test.cpp:13:10:13:16 | call to ymlStep | test.cpp:13:10:13:16 | call to ymlStep | provenance |  |
-| test.cpp:13:10:13:16 | call to ymlStep | test.cpp:15:10:15:10 | y | provenance |  Sink:MaD:643 |
+| test.cpp:13:10:13:16 | call to ymlStep | test.cpp:15:10:15:10 | y | provenance |  Sink:MaD:1 |
 | test.cpp:13:18:13:18 | x | test.cpp:4:5:4:11 | [summary param] 0 in ymlStep | provenance |  |
-| test.cpp:13:18:13:18 | x | test.cpp:13:10:13:16 | call to ymlStep | provenance | MaD:644 |
+| test.cpp:13:18:13:18 | x | test.cpp:13:10:13:16 | call to ymlStep | provenance | MaD:2 |
 nodes
 | asio_streams.cpp:56:18:56:23 | [summary param] *0 in buffer | semmle.label | [summary param] *0 in buffer |
 | asio_streams.cpp:56:18:56:23 | [summary] to write: ReturnValue in buffer | semmle.label | [summary] to write: ReturnValue in buffer |

cpp/ql/test/library-tests/dataflow/fields/ir-path-flow.expected

@@ -848,8 +848,6 @@ edges
 | simple.cpp:120:8:120:8 | *a [i] | simple.cpp:120:10:120:10 | i | provenance |  |
 | struct_init.c:14:24:14:25 | *ab [a] | struct_init.c:14:24:14:25 | *ab [a] | provenance |  |
 | struct_init.c:14:24:14:25 | *ab [a] | struct_init.c:15:8:15:9 | *ab [a] | provenance |  |
-| struct_init.c:14:24:14:25 | *ab [a] | struct_init.c:15:8:15:9 | *ab [a] | provenance |  |
-| struct_init.c:15:8:15:9 | *ab [a] | struct_init.c:15:12:15:12 | a | provenance |  |
 | struct_init.c:15:8:15:9 | *ab [a] | struct_init.c:15:12:15:12 | a | provenance |  |
 | struct_init.c:20:13:20:14 | *definition of ab [a] | struct_init.c:22:8:22:9 | *ab [a] | provenance |  |
 | struct_init.c:20:13:20:14 | *definition of ab [a] | struct_init.c:24:10:24:12 | *& ... [a] | provenance |  |
@@ -1760,8 +1758,6 @@ nodes
 | simple.cpp:120:10:120:10 | i | semmle.label | i |
 | struct_init.c:14:24:14:25 | *ab [a] | semmle.label | *ab [a] |
 | struct_init.c:14:24:14:25 | *ab [a] | semmle.label | *ab [a] |
-| struct_init.c:14:24:14:25 | *ab [a] | semmle.label | *ab [a] |
-| struct_init.c:15:8:15:9 | *ab [a] | semmle.label | *ab [a] |
 | struct_init.c:15:8:15:9 | *ab [a] | semmle.label | *ab [a] |
 | struct_init.c:15:12:15:12 | a | semmle.label | a |
 | struct_init.c:20:13:20:14 | *definition of ab [a] | semmle.label | *definition of ab [a] |

cpp/ql/test/library-tests/dataflow/fields/path-flow.expected

@@ -737,8 +737,6 @@ edges
 | simple.cpp:120:8:120:8 | a [i] | simple.cpp:120:10:120:10 | i | provenance |  |
 | struct_init.c:14:24:14:25 | ab [a] | struct_init.c:14:24:14:25 | ab [a] | provenance |  |
 | struct_init.c:14:24:14:25 | ab [a] | struct_init.c:15:8:15:9 | ab [a] | provenance |  |
-| struct_init.c:14:24:14:25 | ab [a] | struct_init.c:15:8:15:9 | ab [a] | provenance |  |
-| struct_init.c:15:8:15:9 | ab [a] | struct_init.c:15:12:15:12 | a | provenance |  |
 | struct_init.c:15:8:15:9 | ab [a] | struct_init.c:15:12:15:12 | a | provenance |  |
 | struct_init.c:15:8:15:9 | ab [a] | struct_init.c:15:12:15:12 | a | provenance |  |
 | struct_init.c:15:8:15:9 | ab [post update] [a] | struct_init.c:14:24:14:25 | ab [a] | provenance |  |
@@ -1551,8 +1549,6 @@ nodes
 | simple.cpp:120:10:120:10 | i | semmle.label | i |
 | struct_init.c:14:24:14:25 | ab [a] | semmle.label | ab [a] |
 | struct_init.c:14:24:14:25 | ab [a] | semmle.label | ab [a] |
-| struct_init.c:14:24:14:25 | ab [a] | semmle.label | ab [a] |
-| struct_init.c:15:8:15:9 | ab [a] | semmle.label | ab [a] |
 | struct_init.c:15:8:15:9 | ab [a] | semmle.label | ab [a] |
 | struct_init.c:15:8:15:9 | ab [post update] [a] | semmle.label | ab [post update] [a] |
 | struct_init.c:15:12:15:12 | a | semmle.label | a |

cpp/ql/test/library-tests/deduction_guides/missing_template_class.ql

@@ -1,5 +0,0 @@
-import cpp
-
-from DeductionGuide d
-where not exists(d.getTemplateClass())
-select d

cpp/ql/test/library-tests/deduction_guides/template_class.expected

@@ -1,5 +0,0 @@
-| file://:0:0:0:0 | C | test.cpp:4:8:4:8 | C<T> |
-| file://:0:0:0:0 | C | test.cpp:4:8:4:8 | C<T> |
-| test.cpp:5:5:5:5 | (unnamed deduction guide) | test.cpp:4:8:4:8 | C<T> |
-| test.cpp:6:5:6:5 | (unnamed deduction guide) | test.cpp:4:8:4:8 | C<T> |
-| test.cpp:12:1:12:1 | C | test.cpp:4:8:4:8 | C<T> |

cpp/ql/test/library-tests/deduction_guides/template_class.ql

@@ -1,4 +0,0 @@
-import cpp
-
-from DeductionGuide d
-select d, d.getTemplateClass()

cpp/ql/test/library-tests/deduction_guides/test.cpp

@@ -1,21 +0,0 @@
-// semmle-extractor-options: -std=c++20
-
-template<typename T>
-struct C {
-    C(const T);
-    C(char, char);
-};
-
-C(const double)  -> C<int>;
-
-template<typename T>
-C(const T)  -> C<int>;
-
-C(char, char) -> C<char>;
-
-void test() {
-    new C<char>(0);
-    new C<int>(0);
-    new C(0.0f);
-    new C(0.0);
-}

cpp/ql/test/library-tests/ir/ir/PrintAST.expected

@@ -4179,40 +4179,6 @@ destructors_for_temps.cpp:
 #  103|             Type = [IntType] int
 #  103|             ValueCategory = prvalue
 #  104|     getStmt(1): [ReturnStmt] return ...
-generic.c:
-#    1| [TopLevelFunction] void c11_generic_test(unsigned int, int)
-#    1|   <params>: 
-#    1|     getParameter(0): [Parameter] x
-#    1|         Type = [IntType] unsigned int
-#    1|     getParameter(1): [Parameter] y
-#    1|         Type = [IntType] int
-#    1|   getEntryPoint(): [BlockStmt] { ... }
-#    2|     getStmt(0): [DeclStmt] declaration
-#    2|       getDeclarationEntry(0): [VariableDeclarationEntry] definition of r
-#    2|           Type = [IntType] unsigned int
-#    3|     getStmt(1): [ExprStmt] ExprStmt
-#    3|       getExpr(): [AssignExpr] ... = ...
-#    3|           Type = [IntType] unsigned int
-#    3|           ValueCategory = prvalue
-#    3|         getLValue(): [VariableAccess] r
-#    3|             Type = [IntType] unsigned int
-#    3|             ValueCategory = lvalue
-#    3|         getRValue(): [AddExpr] ... + ...
-#    3|             Type = [IntType] unsigned int
-#    3|             ValueCategory = prvalue
-#    3|           getLeftOperand(): [VariableAccess] x
-#    3|               Type = [IntType] unsigned int
-#    3|               ValueCategory = prvalue(load)
-#    3|           getRightOperand(): [Literal] 1
-#    3|               Type = [IntType] int
-#    3|               Value = [Literal] 1
-#    3|               ValueCategory = prvalue
-#    3|           getRightOperand().getFullyConverted(): [CStyleCast] (unsigned int)...
-#    3|               Conversion = [IntegralConversion] integral conversion
-#    3|               Type = [IntType] unsigned int
-#    3|               Value = [CStyleCast] 1
-#    3|               ValueCategory = prvalue
-#    4|     getStmt(2): [ReturnStmt] return ...
 ir.c:
 #    5| [TopLevelFunction] int getX(MyCoords*)
 #    5|   <params>: 

cpp/ql/test/library-tests/ir/ir/aliased_ir.expected

@@ -2958,30 +2958,6 @@ destructors_for_temps.cpp:
 #  102|     v102_9(void)                         = AliasedUse                               : ~m103_26
 #  102|     v102_10(void)                        = ExitFunction                             : 
 
-generic.c:
-#    1| void c11_generic_test(unsigned int, int)
-#    1|   Block 0
-#    1|     v1_1(void)                = EnterFunction          : 
-#    1|     m1_2(unknown)             = AliasedDefinition      : 
-#    1|     m1_3(unknown)             = InitializeNonLocal     : 
-#    1|     m1_4(unknown)             = Chi                    : total:m1_2, partial:m1_3
-#    1|     r1_5(glval<unsigned int>) = VariableAddress[x]     : 
-#    1|     m1_6(unsigned int)        = InitializeParameter[x] : &:r1_5
-#    1|     r1_7(glval<int>)          = VariableAddress[y]     : 
-#    1|     m1_8(int)                 = InitializeParameter[y] : &:r1_7
-#    2|     r2_1(glval<unsigned int>) = VariableAddress[r]     : 
-#    2|     m2_2(unsigned int)        = Uninitialized[r]       : &:r2_1
-#    3|     r3_1(glval<unsigned int>) = VariableAddress[x]     : 
-#    3|     r3_2(unsigned int)        = Load[x]                : &:r3_1, m1_6
-#    3|     r3_3(unsigned int)        = Constant[1]            : 
-#    3|     r3_4(unsigned int)        = Add                    : r3_2, r3_3
-#    3|     r3_5(glval<unsigned int>) = VariableAddress[r]     : 
-#    3|     m3_6(unsigned int)        = Store[r]               : &:r3_5, r3_4
-#    4|     v4_1(void)                = NoOp                   : 
-#    1|     v1_9(void)                = ReturnVoid             : 
-#    1|     v1_10(void)               = AliasedUse             : m1_3
-#    1|     v1_11(void)               = ExitFunction           : 
-
 ir.c:
 #    7| void MyCoordsTest(int)
 #    7|   Block 0

cpp/ql/test/library-tests/ir/ir/generic.c

@@ -1,6 +0,0 @@
-void c11_generic_test(unsigned int x, int y) {
-  unsigned int r;
-  r = _Generic(r, unsigned int: x, int: y) + 1;
-}
-
-// // semmle-extractor-options: -std=c11

cpp/ql/test/library-tests/ir/ir/raw_ir.expected

@@ -2732,29 +2732,6 @@ destructors_for_temps.cpp:
 #  102|     v102_7(void)                         = AliasedUse                               : ~m?
 #  102|     v102_8(void)                         = ExitFunction                             : 
 
-generic.c:
-#    1| void c11_generic_test(unsigned int, int)
-#    1|   Block 0
-#    1|     v1_1(void)                = EnterFunction          : 
-#    1|     mu1_2(unknown)            = AliasedDefinition      : 
-#    1|     mu1_3(unknown)            = InitializeNonLocal     : 
-#    1|     r1_4(glval<unsigned int>) = VariableAddress[x]     : 
-#    1|     mu1_5(unsigned int)       = InitializeParameter[x] : &:r1_4
-#    1|     r1_6(glval<int>)          = VariableAddress[y]     : 
-#    1|     mu1_7(int)                = InitializeParameter[y] : &:r1_6
-#    2|     r2_1(glval<unsigned int>) = VariableAddress[r]     : 
-#    2|     mu2_2(unsigned int)       = Uninitialized[r]       : &:r2_1
-#    3|     r3_1(glval<unsigned int>) = VariableAddress[x]     : 
-#    3|     r3_2(unsigned int)        = Load[x]                : &:r3_1, ~m?
-#    3|     r3_3(unsigned int)        = Constant[1]            : 
-#    3|     r3_4(unsigned int)        = Add                    : r3_2, r3_3
-#    3|     r3_5(glval<unsigned int>) = VariableAddress[r]     : 
-#    3|     mu3_6(unsigned int)       = Store[r]               : &:r3_5, r3_4
-#    4|     v4_1(void)                = NoOp                   : 
-#    1|     v1_8(void)                = ReturnVoid             : 
-#    1|     v1_9(void)                = AliasedUse             : ~m?
-#    1|     v1_10(void)               = ExitFunction           : 
-
 ir.c:
 #    7| void MyCoordsTest(int)
 #    7|   Block 0

cpp/ql/test/library-tests/specifiers2/explicit.expected

@@ -1,12 +0,0 @@
-| cpp20.cpp:17:5:17:5 | (unnamed deduction guide) |  | cpp20.cpp:16:14:16:27 | ... == ... |  |
-| cpp20.cpp:17:5:17:20 | TestExplicitBool |  | cpp20.cpp:16:14:16:27 | ... == ... |  |
-| cpp20.cpp:17:5:17:20 | TestExplicitBool |  | cpp20.cpp:16:14:16:27 | ... == ... | 0 |
-| cpp20.cpp:17:5:17:20 | TestExplicitBool | explicit | cpp20.cpp:16:14:16:27 | ... == ... | 1 |
-| cpp20.cpp:40:1:40:39 | TestExplicitBool2 |  | cpp20.cpp:35:14:35:27 | ... == ... |  |
-| cpp20.cpp:40:23:40:23 | TestExplicitBool2 |  | cpp20.cpp:35:14:35:27 | ... == ... |  |
-| cpp20.cpp:40:23:40:23 | TestExplicitBool2 |  | cpp20.cpp:35:14:35:27 | ... == ... | 0 |
-| cpp20.cpp:40:23:40:23 | TestExplicitBool2 | explicit | cpp20.cpp:35:14:35:27 | ... == ... | 1 |
-| cpp20.cpp:51:5:51:21 | TestExplicitBool3 |  | cpp20.cpp:50:14:50:27 | ... == ... | 0 |
-| cpp20.cpp:51:5:51:21 | TestExplicitBool3 | explicit | cpp20.cpp:50:14:50:27 | ... == ... | 1 |
-| cpp20.cpp:55:1:55:39 | TestExplicitBool3 |  | cpp20.cpp:50:14:50:27 | ... == ... |  |
-| cpp20.cpp:64:5:64:21 | TestExplicitBool4 | explicit | cpp20.cpp:63:14:63:30 | ... == ... | 1 |

cpp/ql/test/library-tests/specifiers2/explicit.ql

@@ -1,8 +0,0 @@
-import cpp
-
-from Function f, string explicit, Expr e, string value
-where
-  (if f.isExplicit() then explicit = "explicit" else explicit = "") and
-  e = f.getExplicitExpr() and
-  if exists(e.getValue()) then value = e.getValue() else value = ""
-select f, explicit, e, value

cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/TaintedAllocationSize/TaintedAllocationSize.expected

@@ -13,26 +13,26 @@ edges
 | test.cpp:133:19:133:32 | *call to getenv | test.cpp:133:14:133:17 | call to atoi | provenance | TaintFunction |
 | test.cpp:148:15:148:18 | call to atol | test.cpp:152:11:152:28 | ... * ... | provenance |  |
 | test.cpp:148:20:148:33 | *call to getenv | test.cpp:148:15:148:18 | call to atol | provenance | TaintFunction |
-| test.cpp:224:8:224:23 | *get_tainted_size | test.cpp:256:9:256:24 | call to get_tainted_size | provenance |  |
-| test.cpp:226:9:226:42 | ... * ... | test.cpp:224:8:224:23 | *get_tainted_size | provenance |  |
-| test.cpp:226:14:226:27 | *call to getenv | test.cpp:226:9:226:42 | ... * ... | provenance | TaintFunction |
-| test.cpp:245:21:245:21 | s | test.cpp:246:21:246:21 | s | provenance |  |
-| test.cpp:252:19:252:52 | ... * ... | test.cpp:254:9:254:18 | local_size | provenance |  |
-| test.cpp:252:19:252:52 | ... * ... | test.cpp:260:11:260:20 | local_size | provenance |  |
-| test.cpp:252:19:252:52 | ... * ... | test.cpp:262:10:262:19 | local_size | provenance |  |
-| test.cpp:252:24:252:37 | *call to getenv | test.cpp:252:19:252:52 | ... * ... | provenance | TaintFunction |
-| test.cpp:262:10:262:19 | local_size | test.cpp:245:21:245:21 | s | provenance |  |
-| test.cpp:265:20:265:27 | *out_size | test.cpp:304:17:304:20 | get_size output argument | provenance |  |
-| test.cpp:265:20:265:27 | *out_size | test.cpp:320:18:320:21 | get_size output argument | provenance |  |
-| test.cpp:266:2:266:32 | ... = ... | test.cpp:265:20:265:27 | *out_size | provenance |  |
-| test.cpp:266:18:266:31 | *call to getenv | test.cpp:266:2:266:32 | ... = ... | provenance | TaintFunction |
-| test.cpp:274:15:274:18 | call to atoi | test.cpp:278:11:278:29 | ... * ... | provenance |  |
-| test.cpp:274:20:274:33 | *call to getenv | test.cpp:274:15:274:18 | call to atoi | provenance | TaintFunction |
-| test.cpp:304:17:304:20 | get_size output argument | test.cpp:306:11:306:28 | ... * ... | provenance |  |
-| test.cpp:320:18:320:21 | get_size output argument | test.cpp:323:10:323:27 | ... * ... | provenance |  |
-| test.cpp:368:13:368:16 | call to atoi | test.cpp:370:35:370:38 | size | provenance |  |
-| test.cpp:368:13:368:16 | call to atoi | test.cpp:371:35:371:38 | size | provenance |  |
-| test.cpp:368:18:368:31 | *call to getenv | test.cpp:368:13:368:16 | call to atoi | provenance | TaintFunction |
+| test.cpp:209:8:209:23 | *get_tainted_size | test.cpp:241:9:241:24 | call to get_tainted_size | provenance |  |
+| test.cpp:211:9:211:42 | ... * ... | test.cpp:209:8:209:23 | *get_tainted_size | provenance |  |
+| test.cpp:211:14:211:27 | *call to getenv | test.cpp:211:9:211:42 | ... * ... | provenance | TaintFunction |
+| test.cpp:230:21:230:21 | s | test.cpp:231:21:231:21 | s | provenance |  |
+| test.cpp:237:19:237:52 | ... * ... | test.cpp:239:9:239:18 | local_size | provenance |  |
+| test.cpp:237:19:237:52 | ... * ... | test.cpp:245:11:245:20 | local_size | provenance |  |
+| test.cpp:237:19:237:52 | ... * ... | test.cpp:247:10:247:19 | local_size | provenance |  |
+| test.cpp:237:24:237:37 | *call to getenv | test.cpp:237:19:237:52 | ... * ... | provenance | TaintFunction |
+| test.cpp:247:10:247:19 | local_size | test.cpp:230:21:230:21 | s | provenance |  |
+| test.cpp:250:20:250:27 | *out_size | test.cpp:289:17:289:20 | get_size output argument | provenance |  |
+| test.cpp:250:20:250:27 | *out_size | test.cpp:305:18:305:21 | get_size output argument | provenance |  |
+| test.cpp:251:2:251:32 | ... = ... | test.cpp:250:20:250:27 | *out_size | provenance |  |
+| test.cpp:251:18:251:31 | *call to getenv | test.cpp:251:2:251:32 | ... = ... | provenance | TaintFunction |
+| test.cpp:259:15:259:18 | call to atoi | test.cpp:263:11:263:29 | ... * ... | provenance |  |
+| test.cpp:259:20:259:33 | *call to getenv | test.cpp:259:15:259:18 | call to atoi | provenance | TaintFunction |
+| test.cpp:289:17:289:20 | get_size output argument | test.cpp:291:11:291:28 | ... * ... | provenance |  |
+| test.cpp:305:18:305:21 | get_size output argument | test.cpp:308:10:308:27 | ... * ... | provenance |  |
+| test.cpp:353:13:353:16 | call to atoi | test.cpp:355:35:355:38 | size | provenance |  |
+| test.cpp:353:13:353:16 | call to atoi | test.cpp:356:35:356:38 | size | provenance |  |
+| test.cpp:353:18:353:31 | *call to getenv | test.cpp:353:13:353:16 | call to atoi | provenance | TaintFunction |
 nodes
 | test.cpp:39:27:39:30 | **argv | semmle.label | **argv |
 | test.cpp:40:16:40:19 | call to atoi | semmle.label | call to atoi |
@@ -52,48 +52,48 @@ nodes
 | test.cpp:148:15:148:18 | call to atol | semmle.label | call to atol |
 | test.cpp:148:20:148:33 | *call to getenv | semmle.label | *call to getenv |
 | test.cpp:152:11:152:28 | ... * ... | semmle.label | ... * ... |
-| test.cpp:224:8:224:23 | *get_tainted_size | semmle.label | *get_tainted_size |
-| test.cpp:226:9:226:42 | ... * ... | semmle.label | ... * ... |
-| test.cpp:226:14:226:27 | *call to getenv | semmle.label | *call to getenv |
-| test.cpp:245:21:245:21 | s | semmle.label | s |
-| test.cpp:246:21:246:21 | s | semmle.label | s |
-| test.cpp:252:19:252:52 | ... * ... | semmle.label | ... * ... |
-| test.cpp:252:24:252:37 | *call to getenv | semmle.label | *call to getenv |
-| test.cpp:254:9:254:18 | local_size | semmle.label | local_size |
-| test.cpp:256:9:256:24 | call to get_tainted_size | semmle.label | call to get_tainted_size |
-| test.cpp:260:11:260:20 | local_size | semmle.label | local_size |
-| test.cpp:262:10:262:19 | local_size | semmle.label | local_size |
-| test.cpp:265:20:265:27 | *out_size | semmle.label | *out_size |
-| test.cpp:266:2:266:32 | ... = ... | semmle.label | ... = ... |
-| test.cpp:266:18:266:31 | *call to getenv | semmle.label | *call to getenv |
-| test.cpp:274:15:274:18 | call to atoi | semmle.label | call to atoi |
-| test.cpp:274:20:274:33 | *call to getenv | semmle.label | *call to getenv |
-| test.cpp:278:11:278:29 | ... * ... | semmle.label | ... * ... |
-| test.cpp:304:17:304:20 | get_size output argument | semmle.label | get_size output argument |
-| test.cpp:306:11:306:28 | ... * ... | semmle.label | ... * ... |
-| test.cpp:320:18:320:21 | get_size output argument | semmle.label | get_size output argument |
-| test.cpp:323:10:323:27 | ... * ... | semmle.label | ... * ... |
-| test.cpp:368:13:368:16 | call to atoi | semmle.label | call to atoi |
-| test.cpp:368:18:368:31 | *call to getenv | semmle.label | *call to getenv |
-| test.cpp:370:35:370:38 | size | semmle.label | size |
-| test.cpp:371:35:371:38 | size | semmle.label | size |
+| test.cpp:209:8:209:23 | *get_tainted_size | semmle.label | *get_tainted_size |
+| test.cpp:211:9:211:42 | ... * ... | semmle.label | ... * ... |
+| test.cpp:211:14:211:27 | *call to getenv | semmle.label | *call to getenv |
+| test.cpp:230:21:230:21 | s | semmle.label | s |
+| test.cpp:231:21:231:21 | s | semmle.label | s |
+| test.cpp:237:19:237:52 | ... * ... | semmle.label | ... * ... |
+| test.cpp:237:24:237:37 | *call to getenv | semmle.label | *call to getenv |
+| test.cpp:239:9:239:18 | local_size | semmle.label | local_size |
+| test.cpp:241:9:241:24 | call to get_tainted_size | semmle.label | call to get_tainted_size |
+| test.cpp:245:11:245:20 | local_size | semmle.label | local_size |
+| test.cpp:247:10:247:19 | local_size | semmle.label | local_size |
+| test.cpp:250:20:250:27 | *out_size | semmle.label | *out_size |
+| test.cpp:251:2:251:32 | ... = ... | semmle.label | ... = ... |
+| test.cpp:251:18:251:31 | *call to getenv | semmle.label | *call to getenv |
+| test.cpp:259:15:259:18 | call to atoi | semmle.label | call to atoi |
+| test.cpp:259:20:259:33 | *call to getenv | semmle.label | *call to getenv |
+| test.cpp:263:11:263:29 | ... * ... | semmle.label | ... * ... |
+| test.cpp:289:17:289:20 | get_size output argument | semmle.label | get_size output argument |
+| test.cpp:291:11:291:28 | ... * ... | semmle.label | ... * ... |
+| test.cpp:305:18:305:21 | get_size output argument | semmle.label | get_size output argument |
+| test.cpp:308:10:308:27 | ... * ... | semmle.label | ... * ... |
+| test.cpp:353:13:353:16 | call to atoi | semmle.label | call to atoi |
+| test.cpp:353:18:353:31 | *call to getenv | semmle.label | *call to getenv |
+| test.cpp:355:35:355:38 | size | semmle.label | size |
+| test.cpp:356:35:356:38 | size | semmle.label | size |
 subpaths
 #select
-| test.cpp:43:31:43:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:43:38:43:44 | tainted | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:44:31:44:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:44:38:44:63 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:46:31:46:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:46:38:46:63 | ... + ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:49:25:49:30 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:49:32:49:35 | size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:50:17:50:30 | new[] | test.cpp:39:27:39:30 | **argv | test.cpp:50:17:50:30 | size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:53:21:53:27 | call to realloc | test.cpp:39:27:39:30 | **argv | test.cpp:53:35:53:60 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:128:17:128:22 | call to malloc | test.cpp:124:18:124:31 | *call to getenv | test.cpp:128:24:128:41 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:124:18:124:31 | *call to getenv | user input (an environment variable) |
-| test.cpp:135:3:135:8 | call to malloc | test.cpp:133:19:133:32 | *call to getenv | test.cpp:135:10:135:27 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:133:19:133:32 | *call to getenv | user input (an environment variable) |
-| test.cpp:152:4:152:9 | call to malloc | test.cpp:148:20:148:33 | *call to getenv | test.cpp:152:11:152:28 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:148:20:148:33 | *call to getenv | user input (an environment variable) |
-| test.cpp:246:14:246:19 | call to malloc | test.cpp:252:24:252:37 | *call to getenv | test.cpp:246:21:246:21 | s | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:252:24:252:37 | *call to getenv | user input (an environment variable) |
-| test.cpp:254:2:254:7 | call to malloc | test.cpp:252:24:252:37 | *call to getenv | test.cpp:254:9:254:18 | local_size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:252:24:252:37 | *call to getenv | user input (an environment variable) |
-| test.cpp:256:2:256:7 | call to malloc | test.cpp:226:14:226:27 | *call to getenv | test.cpp:256:9:256:24 | call to get_tainted_size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:226:14:226:27 | *call to getenv | user input (an environment variable) |
-| test.cpp:260:2:260:9 | call to my_alloc | test.cpp:252:24:252:37 | *call to getenv | test.cpp:260:11:260:20 | local_size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:252:24:252:37 | *call to getenv | user input (an environment variable) |
-| test.cpp:278:4:278:9 | call to malloc | test.cpp:274:20:274:33 | *call to getenv | test.cpp:278:11:278:29 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:274:20:274:33 | *call to getenv | user input (an environment variable) |
-| test.cpp:306:4:306:9 | call to malloc | test.cpp:266:18:266:31 | *call to getenv | test.cpp:306:11:306:28 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:266:18:266:31 | *call to getenv | user input (an environment variable) |
-| test.cpp:323:3:323:8 | call to malloc | test.cpp:266:18:266:31 | *call to getenv | test.cpp:323:10:323:27 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:266:18:266:31 | *call to getenv | user input (an environment variable) |
-| test.cpp:370:25:370:33 | call to MyMalloc1 | test.cpp:368:18:368:31 | *call to getenv | test.cpp:370:35:370:38 | size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:368:18:368:31 | *call to getenv | user input (an environment variable) |
-| test.cpp:371:25:371:33 | call to MyMalloc2 | test.cpp:368:18:368:31 | *call to getenv | test.cpp:371:35:371:38 | size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:368:18:368:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:43:31:43:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:43:38:43:44 | tainted | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:44:31:44:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:44:38:44:63 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:46:31:46:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:46:38:46:63 | ... + ... | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:49:25:49:30 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:49:32:49:35 | size | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:50:17:50:30 | new[] | test.cpp:39:27:39:30 | **argv | test.cpp:50:17:50:30 | size | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:53:21:53:27 | call to realloc | test.cpp:39:27:39:30 | **argv | test.cpp:53:35:53:60 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:128:17:128:22 | call to malloc | test.cpp:124:18:124:31 | *call to getenv | test.cpp:128:24:128:41 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:124:18:124:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:135:3:135:8 | call to malloc | test.cpp:133:19:133:32 | *call to getenv | test.cpp:135:10:135:27 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:133:19:133:32 | *call to getenv | user input (an environment variable) |
+| test.cpp:152:4:152:9 | call to malloc | test.cpp:148:20:148:33 | *call to getenv | test.cpp:152:11:152:28 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:148:20:148:33 | *call to getenv | user input (an environment variable) |
+| test.cpp:231:14:231:19 | call to malloc | test.cpp:237:24:237:37 | *call to getenv | test.cpp:231:21:231:21 | s | This allocation size is derived from $@ and might overflow. | test.cpp:237:24:237:37 | *call to getenv | user input (an environment variable) |
+| test.cpp:239:2:239:7 | call to malloc | test.cpp:237:24:237:37 | *call to getenv | test.cpp:239:9:239:18 | local_size | This allocation size is derived from $@ and might overflow. | test.cpp:237:24:237:37 | *call to getenv | user input (an environment variable) |
+| test.cpp:241:2:241:7 | call to malloc | test.cpp:211:14:211:27 | *call to getenv | test.cpp:241:9:241:24 | call to get_tainted_size | This allocation size is derived from $@ and might overflow. | test.cpp:211:14:211:27 | *call to getenv | user input (an environment variable) |
+| test.cpp:245:2:245:9 | call to my_alloc | test.cpp:237:24:237:37 | *call to getenv | test.cpp:245:11:245:20 | local_size | This allocation size is derived from $@ and might overflow. | test.cpp:237:24:237:37 | *call to getenv | user input (an environment variable) |
+| test.cpp:263:4:263:9 | call to malloc | test.cpp:259:20:259:33 | *call to getenv | test.cpp:263:11:263:29 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:259:20:259:33 | *call to getenv | user input (an environment variable) |
+| test.cpp:291:4:291:9 | call to malloc | test.cpp:251:18:251:31 | *call to getenv | test.cpp:291:11:291:28 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:251:18:251:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:308:3:308:8 | call to malloc | test.cpp:251:18:251:31 | *call to getenv | test.cpp:308:10:308:27 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:251:18:251:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:355:25:355:33 | call to MyMalloc1 | test.cpp:353:18:353:31 | *call to getenv | test.cpp:355:35:355:38 | size | This allocation size is derived from $@ and might overflow. | test.cpp:353:18:353:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:356:25:356:33 | call to MyMalloc2 | test.cpp:353:18:353:31 | *call to getenv | test.cpp:356:35:356:38 | size | This allocation size is derived from $@ and might overflow. | test.cpp:353:18:353:31 | *call to getenv | user input (an environment variable) |

cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/TaintedAllocationSize/test.cpp

@@ -40,10 +40,10 @@ int main(int argc, char **argv) {
 	int tainted = atoi(argv[1]);
 
 	MyStruct *arr1 = (MyStruct *)malloc(sizeof(MyStruct)); // GOOD
-	MyStruct *arr2 = (MyStruct *)malloc(tainted); // BAD
+	MyStruct *arr2 = (MyStruct *)malloc(tainted); // DUBIOUS (not multiplied by anything)
 	MyStruct *arr3 = (MyStruct *)malloc(tainted * sizeof(MyStruct)); // BAD
 	MyStruct *arr4 = (MyStruct *)malloc(getTainted() * sizeof(MyStruct)); // BAD [NOT DETECTED]
-	MyStruct *arr5 = (MyStruct *)malloc(sizeof(MyStruct) + tainted); // BAD
+	MyStruct *arr5 = (MyStruct *)malloc(sizeof(MyStruct) + tainted); // DUBIOUS (not multiplied by anything)
 
 	int size = tainted * 8;
 	char *chars1 = (char *)malloc(size); // BAD
@@ -180,21 +180,6 @@ void more_bounded_tests() {
 		}
 	}
 
-	{
-		int size = atoi(getenv("USER"));
-		int size2 = size % 100;
-		malloc(size2 * sizeof(int)); // GOOD
-	}
-
-	{
-		int size = atoi(getenv("USER"));
-
-		if (size % 100)
-		{
-			malloc(size * sizeof(int)); // BAD [NOT DETECTED]
-		}
-	}
-
 	{
 		int size = atoi(getenv("USER"));
 

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -7,7 +7,6 @@ using System.Xml;
 using Microsoft.Build.Construction;
 using Semmle.Util;
 using Semmle.Autobuild.Shared;
-using Semmle.Util.Logging;
 
 namespace Semmle.Autobuild.CSharp.Tests
 {
@@ -204,7 +203,7 @@ namespace Semmle.Autobuild.CSharp.Tests
                 throw new ArgumentException($"Missing CreateDirectory, {path}");
         }
 
-        public void DownloadFile(string address, string fileName, ILogger logger)
+        public void DownloadFile(string address, string fileName)
         {
             if (!DownloadFiles.Contains((address, fileName)))
                 throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");

csharp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -7,7 +7,6 @@ using System.Linq;
 using Microsoft.Build.Construction;
 using System.Xml;
 using System.IO;
-using Semmle.Util.Logging;
 
 namespace Semmle.Autobuild.Cpp.Tests
 {
@@ -190,7 +189,7 @@ namespace Semmle.Autobuild.Cpp.Tests
                 throw new ArgumentException($"Missing CreateDirectory, {path}");
         }
 
-        public void DownloadFile(string address, string fileName, ILogger logger)
+        public void DownloadFile(string address, string fileName)
         {
             if (!DownloadFiles.Contains((address, fileName)))
                 throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");

csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs

@@ -157,8 +157,7 @@ namespace Semmle.Autobuild.Shared
             BuildScript.DownloadFile(
                 FileUtils.NugetExeUrl,
                 path,
-                e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}"),
-                builder.Logger)
+                e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}"))
             &
             BuildScript.Create(_ =>
             {

csharp/documentation/library-coverage/coverage.csv

@@ -1,5 +1,5 @@
-package,sink,source,summary,sink:code-injection,sink:encryption-decryptor,sink:encryption-encryptor,sink:encryption-keyprop,sink:encryption-symmetrickey,sink:file-content-store,sink:html-injection,sink:js-injection,sink:log-injection,sink:sql-injection,source:commandargs,source:database,source:environment,source:file,source:file-write,source:remote,source:stdin,source:windows-registry,summary:taint,summary:value
-Amazon.Lambda.APIGatewayEvents,,6,,,,,,,,,,,,,,,,,6,,,,
+package,sink,source,summary,sink:code-injection,sink:encryption-decryptor,sink:encryption-encryptor,sink:encryption-keyprop,sink:encryption-symmetrickey,sink:file-content-store,sink:html-injection,sink:js-injection,sink:log-injection,sink:sql-injection,source:commandargs,source:database,source:environment,source:file,source:file-write,source:local,source:remote,source:windows-registry,summary:taint,summary:value
+Amazon.Lambda.APIGatewayEvents,,6,,,,,,,,,,,,,,,,,,6,,,
 Amazon.Lambda.Core,10,,,,,,,,,,,10,,,,,,,,,,,
 Dapper,55,42,1,,,,,,,,,,55,,42,,,,,,,,1
 ILCompiler,,,123,,,,,,,,,,,,,,,,,,,123,
@@ -10,7 +10,7 @@ Internal.IL,,,46,,,,,,,,,,,,,,,,,,,44,2
 Internal.Pgo,,,9,,,,,,,,,,,,,,,,,,,8,1
 Internal.TypeSystem,,,315,,,,,,,,,,,,,,,,,,,299,16
 JsonToItemsTaskFactory,,,10,,,,,,,,,,,,,,,,,,,10,
-Microsoft.Android.Build,,1,16,,,,,,,,,,,,,1,,,,,,16,
+Microsoft.Android.Build,,,16,,,,,,,,,,,,,,,,,,,16,
 Microsoft.Apple.Build,,,8,,,,,,,,,,,,,,,,,,,8,
 Microsoft.ApplicationBlocks.Data,28,,,,,,,,,,,,28,,,,,,,,,,
 Microsoft.CSharp,,,13,,,,,,,,,,,,,,,,,,,13,
@@ -19,9 +19,9 @@ Microsoft.DotNet.Build.Tasks,,,6,,,,,,,,,,,,,,,,,,,6,
 Microsoft.EntityFrameworkCore,6,,12,,,,,,,,,,6,,,,,,,,,,12
 Microsoft.Extensions.Caching.Distributed,,,10,,,,,,,,,,,,,,,,,,,10,
 Microsoft.Extensions.Caching.Memory,,,39,,,,,,,,,,,,,,,,,,,38,1
-Microsoft.Extensions.Configuration,,3,90,,,,,,,,,,,,,3,,,,,,89,1
+Microsoft.Extensions.Configuration,,2,90,,,,,,,,,,,,,2,,,,,,89,1
 Microsoft.Extensions.DependencyInjection,,,134,,,,,,,,,,,,,,,,,,,133,1
-Microsoft.Extensions.DependencyModel,,1,18,,,,,,,,,,,,,1,,,,,,18,
+Microsoft.Extensions.DependencyModel,,,18,,,,,,,,,,,,,,,,,,,18,
 Microsoft.Extensions.Diagnostics.Metrics,,,15,,,,,,,,,,,,,,,,,,,15,
 Microsoft.Extensions.FileProviders,,,15,,,,,,,,,,,,,,,,,,,15,
 Microsoft.Extensions.FileSystemGlobbing,,,18,,,,,,,,,,,,,,,,,,,16,2
@@ -41,5 +41,5 @@ MySql.Data.MySqlClient,48,,,,,,,,,,,,48,,,,,,,,,,
 Newtonsoft.Json,,,91,,,,,,,,,,,,,,,,,,,73,18
 ServiceStack,194,,7,27,,,,,75,,,,92,,,,,,,,,7,
 SourceGenerators,,,5,,,,,,,,,,,,,,,,,,,5,
-System,54,47,10626,,6,5,5,,,4,1,,33,2,,6,15,17,4,3,,8721,1905
+System,60,44,10614,,7,6,5,,,4,5,,33,2,,3,15,17,3,4,,8709,1905
 Windows.Security.Cryptography.Core,1,,,,,,,1,,,,,,,,,,,,,,,

csharp/documentation/library-coverage/coverage.rst

@@ -8,7 +8,7 @@ C# framework & library support
 
    Framework / library,Package,Flow sources,Taint & value steps,Sinks (total),`CWE-079` :sub:`Cross-site scripting`
    `ServiceStack <https://servicestack.net/>`_,"``ServiceStack.*``, ``ServiceStack``",,7,194,
-   System,"``System.*``, ``System``",47,10626,54,5
-   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.DotNet.Build.Tasks``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",57,1821,148,
-   Totals,,104,12454,396,5
+   System,"``System.*``, ``System``",44,10614,60,9
+   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.DotNet.Build.Tasks``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",54,1821,148,
+   Totals,,98,12442,402,9
 

csharp/downgrades/15b989afd2bfc4743536fdb0958c1d8177a32600/upgrade.properties

@@ -1,2 +0,0 @@
-description: Add unique constraint on preprocessor directive and compilation pairs
-compatibility: backwards

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs

@@ -189,13 +189,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// (together with used package information) required for compilation.
         /// </summary>
         /// <returns>True if parsing succeeds, otherwise false.</returns>
-        public bool TryParse(string json, string jsonPath)
+        public bool TryParse(string json)
         {
             try
             {
                 var obj = JObject.Parse(json);
-                AddPackageDependencies(obj, jsonPath);
-                AddFrameworkDependencies(obj, jsonPath);
+                AddPackageDependencies(obj, json);
+                AddFrameworkDependencies(obj, json);
                 return true;
             }
             catch (Exception e)
@@ -228,7 +228,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         {
             if (TryReadAllText(asset, logger, out var json))
             {
-                TryParse(json, asset);
+                TryParse(json);
             }
         }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -248,8 +248,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                     var downloadDotNetInstallSh = BuildScript.DownloadFile(
                         "https://dot.net/v1/dotnet-install.sh",
                         dotnetInstallPath,
-                        e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"),
-                        logger);
+                        e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"));
 
                     var chmod = new CommandBuilder(actions).
                         RunCommand("chmod").

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs

@@ -145,7 +145,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
             Directory.CreateDirectory(directory);
             logger.LogInfo("Attempting to download nuget.exe");
-            FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget, logger);
+            FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget);
             logger.LogInfo($"Downloaded nuget.exe to {nuget}");
             return nuget;
         }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -44,9 +44,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             this.logger = logger;
             this.compilationInfoContainer = compilationInfoContainer;
 
-            PackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("packages"), "package", logger);
-            legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("legacypackages"), "legacy package", logger);
-            missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("missingpackages"), "missing package", logger);
+            PackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath(fileProvider.SourceDir.FullName, "packages"), "package", logger);
+            legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath(fileProvider.SourceDir.FullName, "legacypackages"), "legacy package", logger);
+            missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath(fileProvider.SourceDir.FullName, "missingpackages"), "missing package", logger);
         }
 
         public string? TryRestore(string package)
@@ -338,7 +338,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
 
             logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
-            using var tempDir = new TemporaryDirectory(ComputeTempDirectoryPath("nugetconfig"), "generated nuget config", logger);
+            using var tempDir = new TemporaryDirectory(ComputeTempDirectoryPath(fileProvider.SourceDir.FullName, "nugetconfig"), "generated nuget config", logger);
             var nugetConfig = fallbackNugetFeeds is null
                 ? GetNugetConfig()
                 : CreateFallbackNugetConfig(fallbackNugetFeeds, tempDir.DirInfo.FullName);
@@ -667,7 +667,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                     "Found unreachable Nuget feed in C# analysis with build-mode 'none'",
                     visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
                     markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
-                    severity: DiagnosticMessage.TspSeverity.Note
+                    severity: DiagnosticMessage.TspSeverity.Warning
                 ));
             }
             compilationInfoContainer.CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));
@@ -771,19 +771,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         }
 
         /// <summary>
-        /// Returns the full path to a temporary directory with the given subfolder name.
-        /// </summary>
-        private static string ComputeTempDirectoryPath(string subfolderName)
-        {
-            return Path.Combine(FileUtils.GetTemporaryWorkingDirectory(out _), subfolderName);
-        }
-
-        /// <summary>
-        /// Computes a unique temporary directory path based on the source directory and the subfolder name.
+        /// Computes a unique temp directory for the packages associated
+        /// with this source tree. Use a SHA1 of the directory name.
         /// </summary>
+        /// <returns>The full path of the temp directory.</returns>
         private static string ComputeTempDirectoryPath(string srcDir, string subfolderName)
         {
-            return Path.Combine(FileUtils.GetTemporaryWorkingDirectory(out _), FileUtils.ComputeHash(srcDir), subfolderName);
+            var bytes = Encoding.Unicode.GetBytes(srcDir);
+            var sha = SHA1.HashData(bytes);
+            var sb = new StringBuilder();
+            foreach (var b in sha.Take(8))
+                sb.AppendFormat("{0:x2}", b);
+
+            return Path.Combine(FileUtils.GetTemporaryWorkingDirectory(out _), sb.ToString(), subfolderName);
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs

@@ -83,7 +83,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 var targetDir = GetTemporaryWorkingDirectory(FileType.ToLowerInvariant());
 
                 return groupedFiles
-                    .SelectMany(group => sourceGenerator.RunSourceGenerator(group.Value, group.Key, references, targetDir, fileProvider.SourceDir.FullName));
+                    .SelectMany(group => sourceGenerator.RunSourceGenerator(group.Value, group.Key, references, targetDir));
             }
             catch (Exception ex)
             {

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorWrapper/DotnetSourceGeneratorWrapper.cs

@@ -33,12 +33,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         protected abstract void GenerateAnalyzerConfig(IEnumerable<string> additionalFiles, string csprojFile, string analyzerConfigPath);
 
-        public IEnumerable<string> RunSourceGenerator(IEnumerable<string> additionalFiles, string csprojFile, IEnumerable<string> references, string targetDir, string sourceDir)
+        public IEnumerable<string> RunSourceGenerator(IEnumerable<string> additionalFiles, string csprojFile, IEnumerable<string> references, string targetDir)
         {
             try
             {
-                var relativePathToCsProj = Path.GetRelativePath(sourceDir, csprojFile);
-                var name = FileUtils.ComputeHash($"{relativePathToCsProj}\n{this.GetType().Name}");
+                var name = Guid.NewGuid().ToString("N").ToUpper();
                 using var tempDir = new TemporaryDirectory(Path.Join(FileUtils.GetTemporaryWorkingDirectory(out _), "source-generator"), "source generator temporary", logger);
                 var analyzerConfigPath = Path.Combine(tempDir.DirInfo.FullName, $"{name}.txt");
                 var dllPath = Path.Combine(tempDir.DirInfo.FullName, $"{name}.dll");

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs

@@ -3,7 +3,6 @@ using System.Collections.Concurrent;
 using System.IO;
 using System.Linq;
 using Microsoft.CodeAnalysis;
-using Semmle.Extraction.Entities;
 using Semmle.Util;
 
 namespace Semmle.Extraction.CSharp.Entities
@@ -90,21 +89,13 @@ namespace Semmle.Extraction.CSharp.Entities
             trapFile.compilation_finished(this, (float)p.Total.Cpu.TotalSeconds, (float)p.Total.Elapsed.TotalSeconds);
         }
 
-        public void PopulateAggregatedMessages()
-        {
-            ExtractionMessage.groupedMessageCounts.ForEach(pair =>
-            {
-                Context.TrapWriter.Writer.compilation_info(this, $"Extractor message count for group '{pair.Key}'", pair.Value.ToString());
-            });
-        }
-
         public override void WriteId(EscapingTextWriter trapFile)
         {
             trapFile.Write(hashCode);
             trapFile.Write(";compilation");
         }
 
-        public override Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException();
+        public override Location ReportingLocation => throw new NotImplementedException();
 
         public override bool NeedsPopulation => Context.IsAssemblyScope;
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs

@@ -21,6 +21,7 @@ namespace Semmle.Extraction.CSharp.Entities
 
         protected override void Populate(TextWriter trapFile)
         {
+            // The below doesn't limit the extractor messages to the exact limit, but it's good enough.
             var key = diagnostic.Id;
             var messageCount = compilation.messageCounts.AddOrUpdate(key, 1, (_, c) => c + 1);
             if (messageCount > limit)

csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs

@@ -21,7 +21,7 @@ namespace Semmle.Extraction.CSharp.Entities
         public override Microsoft.CodeAnalysis.Location ReportingLocation =>
             IsCompilerGeneratedDelegate()
                 ? Symbol.ContainingType.GetSymbolLocation()
-                : BodyDeclaringSymbol.GetSymbolLocation();
+                : Symbol.GetSymbolLocation();
 
         public override bool NeedsPopulation => base.NeedsPopulation || IsCompilerGeneratedDelegate();
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/ElifDirective.cs

@@ -20,7 +20,6 @@ namespace Semmle.Extraction.CSharp.Entities
         public override void WriteId(EscapingTextWriter trapFile)
         {
             trapFile.WriteSubId(Context.CreateLocation(ReportingLocation));
-            trapFile.WriteSubId(start);
             trapFile.Write(Symbol.IsActive);
             trapFile.Write(',');
             trapFile.Write(Symbol.BranchTaken);

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/ElseDirective.cs

@@ -18,7 +18,6 @@ namespace Semmle.Extraction.CSharp.Entities
         public override void WriteId(EscapingTextWriter trapFile)
         {
             trapFile.WriteSubId(Context.CreateLocation(ReportingLocation));
-            trapFile.WriteSubId(start);
             trapFile.Write(Symbol.IsActive);
             trapFile.Write(',');
             trapFile.Write(Symbol.BranchTaken);

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/EndIfDirective.cs

@@ -13,14 +13,6 @@ namespace Semmle.Extraction.CSharp.Entities
             this.start = start;
         }
 
-        public override void WriteId(EscapingTextWriter trapFile)
-        {
-            trapFile.WriteSubId(Context.CreateLocation(ReportingLocation));
-            trapFile.WriteSubId(start);
-            trapFile.Write(Symbol.IsActive);
-            trapFile.Write(";trivia");
-        }
-
         protected override void PopulatePreprocessor(TextWriter trapFile)
         {
             trapFile.directive_endifs(this, start);

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -190,29 +190,37 @@ namespace Semmle.Extraction.CSharp
                 var transformedSourcePath = PathTransformer.Transform(sourcePath);
 
                 var trapPath = transformedSourcePath.GetTrapPath(Logger, options.TrapCompression);
+                var upToDate = false;
+
+                // compilation.Clone() is used to allow symbols to be garbage collected.
                 using var trapWriter = transformedSourcePath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
 
+                upToDate = FileIsUpToDate(sourcePath, trapWriter.TrapFile);
+
                 var currentTaskId = IncrementTaskCount();
                 ReportProgressTaskStarted(currentTaskId, sourcePath);
 
-                var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
-                // Ensure that the file itself is populated in case the source file is totally empty
-                var root = tree.GetRoot();
-                Entities.File.Create(cx, root.SyntaxTree.FilePath);
-
-                var csNode = (CSharpSyntaxNode)root;
-                var directiveVisitor = new DirectiveVisitor(cx);
-                csNode.Accept(directiveVisitor);
-                foreach (var branch in directiveVisitor.BranchesTaken)
+                if (!upToDate)
                 {
-                    cx.TrapStackSuffix.Add(branch);
-                }
-                csNode.Accept(new CompilationUnitVisitor(cx));
-                cx.PopulateAll();
-                CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
-                cx.PopulateAll();
+                    var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
+                    // Ensure that the file itself is populated in case the source file is totally empty
+                    var root = tree.GetRoot();
+                    Entities.File.Create(cx, root.SyntaxTree.FilePath);
 
-                ReportProgressTaskDone(currentTaskId, sourcePath, trapPath, stopwatch.Elapsed, AnalysisAction.Extracted);
+                    var csNode = (CSharpSyntaxNode)root;
+                    var directiveVisitor = new DirectiveVisitor(cx);
+                    csNode.Accept(directiveVisitor);
+                    foreach (var branch in directiveVisitor.BranchesTaken)
+                    {
+                        cx.TrapStackSuffix.Add(branch);
+                    }
+                    csNode.Accept(new CompilationUnitVisitor(cx));
+                    cx.PopulateAll();
+                    CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
+                    cx.PopulateAll();
+                }
+
+                ReportProgressTaskDone(currentTaskId, sourcePath, trapPath, stopwatch.Elapsed, upToDate ? AnalysisAction.UpToDate : AnalysisAction.Extracted);
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
@@ -250,8 +258,6 @@ namespace Semmle.Extraction.CSharp
 
         public void LogPerformance(Entities.PerformanceMetrics p) => compilationEntity.PopulatePerformance(p);
 
-        public void ExtractAggregatedMessages() => compilationEntity.PopulateAggregatedMessages();
-
 #nullable restore warnings
 
         /// <summary>
@@ -262,6 +268,12 @@ namespace Semmle.Extraction.CSharp
             extractionTasks.Add(() => DoAnalyseCompilation());
         }
 
+        private static bool FileIsUpToDate(string src, string dest)
+        {
+            return File.Exists(dest) &&
+                File.GetLastWriteTime(dest) >= File.GetLastWriteTime(src);
+        }
+
         private static void AnalyseNamespace(Context cx, INamespaceSymbol ns)
         {
             foreach (var memberNamespace in ns.GetNamespaceMembers())

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -458,7 +458,6 @@ namespace Semmle.Extraction.CSharp
 
             sw.Restart();
             analyser.PerformExtraction(options.Threads);
-            analyser.ExtractAggregatedMessages();
             sw.Stop();
             var cpuTime2 = currentProcess.TotalProcessorTime;
             var userTime2 = currentProcess.UserProcessorTime;

csharp/extractor/Semmle.Extraction.Tests/Assets.cs

@@ -16,7 +16,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsJson1;
 
             // Execute
-            var success = assets.TryParse(json, "");
+            var success = assets.TryParse(json);
 
             // Verify
             Assert.True(success);
@@ -46,7 +46,7 @@ namespace Semmle.Extraction.Tests
             var json = "garbage data";
 
             // Execute
-            var success = assets.TryParse(json, "");
+            var success = assets.TryParse(json);
 
             // Verify
             Assert.False(success);
@@ -61,7 +61,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNet70;
 
             // Execute
-            var success = assets.TryParse(json, "");
+            var success = assets.TryParse(json);
 
             // Verify
             Assert.True(success);
@@ -91,7 +91,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNet48;
 
             // Execute
-            var success = assets.TryParse(json, "");
+            var success = assets.TryParse(json);
 
             // Verify
             Assert.True(success);
@@ -117,7 +117,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNetstandard21;
 
             // Execute
-            var success = assets.TryParse(json, "");
+            var success = assets.TryParse(json);
 
             // Verify
             Assert.True(success);
@@ -144,7 +144,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNetstandard16;
 
             // Execute
-            var success = assets.TryParse(json, "");
+            var success = assets.TryParse(json);
 
             // Verify
             Assert.True(success);
@@ -175,7 +175,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNetcoreapp20;
 
             // Execute
-            var success = assets.TryParse(json, "");
+            var success = assets.TryParse(json);
 
             // Verify
             Assert.True(success);
@@ -202,7 +202,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNetcoreapp31;
 
             // Execute
-            var success = assets.TryParse(json, "");
+            var success = assets.TryParse(json);
 
             // Verify
             Assert.True(success);

csharp/extractor/Semmle.Extraction/BUILD.bazel

@@ -26,7 +26,6 @@ codeql_csharp_library(
         ],
         "//conditions:default": [],
     }),
-    internals_visible_to = ["Semmle.Extraction.CSharp"],
     visibility = ["//csharp:__subpackages__"],
     deps = [
         "//csharp/extractor/Semmle.Util",

csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs

@@ -1,5 +1,4 @@
-using System.Collections.Concurrent;
-using System.IO;
+using System.IO;
 using System.Threading;
 using Semmle.Util;
 
@@ -8,45 +7,30 @@ namespace Semmle.Extraction.Entities
     internal class ExtractionMessage : FreshEntity
     {
         private static readonly int limit = EnvironmentVariables.TryGetExtractorNumberOption<int>("MESSAGE_LIMIT") ?? 10000;
-
-        internal static readonly ConcurrentDictionary<string, int> groupedMessageCounts = [];
         private static int messageCount = 0;
 
         private readonly Message msg;
-        private readonly bool bypassLimit;
 
-        public ExtractionMessage(Context cx, Message msg) : this(cx, msg, bypassLimit: false)
+        public ExtractionMessage(Context cx, Message msg) : base(cx)
         {
-        }
-
-        private ExtractionMessage(Context cx, Message msg, bool bypassLimit) : base(cx)
-        {
-            this.bypassLimit = bypassLimit;
             this.msg = msg;
             TryPopulate();
         }
 
         protected override void Populate(TextWriter trapFile)
         {
-            // For the time being we're counting the number of messages per severity, we could introduce other groupings in the future
-            var key = msg.Severity.ToString();
-            groupedMessageCounts.AddOrUpdate(key, 1, (_, c) => c + 1);
-
-            if (!bypassLimit)
+            // The below doesn't limit the extractor messages to the exact limit, but it's good enough.
+            Interlocked.Increment(ref messageCount);
+            if (messageCount > limit)
             {
-                var val = Interlocked.Increment(ref messageCount);
-                if (val > limit)
+                if (messageCount == limit + 1)
                 {
-                    if (val == limit + 1)
-                    {
-                        Context.ExtractionContext.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}");
-                        _ = new ExtractionMessage(Context, new Message($"Stopped logging extractor messages after reaching {limit}", null, null, null, Util.Logging.Severity.Warning), bypassLimit: true);
-                    }
-                    return;
+                    Context.ExtractionContext.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}");
                 }
+                return;
             }
 
-            trapFile.extractor_messages(this, msg.Severity, msg.Text, msg.EntityText ?? string.Empty,
+            trapFile.extractor_messages(this, msg.Severity, "C# extractor", msg.Text, msg.EntityText ?? string.Empty,
                 msg.Location ?? Context.CreateLocation(), msg.StackTrace ?? string.Empty);
         }
     }

csharp/extractor/Semmle.Extraction/Semmle.Extraction.csproj

@@ -5,8 +5,6 @@
 	</PropertyGroup>
 	<ItemGroup>
 		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
-
-		<InternalsVisibleTo Include="Semmle.Extraction.CSharp" />
 	</ItemGroup>
   <Import Project="..\..\.paket\Paket.Restore.targets" />
 </Project>

csharp/extractor/Semmle.Extraction/TrapWriter.cs

@@ -194,11 +194,8 @@ namespace Semmle.Extraction
                     var hash = FileUtils.ComputeFileHash(tmpFile);
                     if (existingHash != hash)
                     {
-                        var extension = TrapExtension(trapCompression);
-                        var root = TrapFile[..^extension.Length]; // Remove trailing ".trap", ".trap.gz", or ".trap.br"
-                        var newTrapName = $"{root}-{hash}{extension}";
-                        logger.LogInfo($"Identical trap file for {TrapFile} already exists, renaming to {newTrapName}");
-                        if (TryMove(tmpFile, $"{newTrapName}"))
+                        var root = TrapFile.Substring(0, TrapFile.Length - 8); // Remove trailing ".trap.gz"
+                        if (TryMove(tmpFile, $"{root}-{hash}.trap{TrapExtension(trapCompression)}"))
                             return;
                     }
                     logger.LogInfo($"Identical trap file for {TrapFile} already exists");
@@ -220,16 +217,16 @@ namespace Semmle.Extraction
         {
             switch (compression)
             {
-                case CompressionMode.None: return ".trap";
-                case CompressionMode.Gzip: return ".trap.gz";
-                case CompressionMode.Brotli: return ".trap.br";
+                case CompressionMode.None: return "";
+                case CompressionMode.Gzip: return ".gz";
+                case CompressionMode.Brotli: return ".br";
                 default: throw new ArgumentOutOfRangeException(nameof(compression), compression, "Unsupported compression type");
             }
         }
 
         public static string TrapPath(ILogger logger, string? folder, PathTransformer.ITransformedPath path, TrapWriter.CompressionMode trapCompression)
         {
-            var filename = $"{path.Value}{TrapExtension(trapCompression)}";
+            var filename = $"{path.Value}.trap{TrapExtension(trapCompression)}";
             if (string.IsNullOrEmpty(folder))
                 folder = Directory.GetCurrentDirectory();
 

csharp/extractor/Semmle.Extraction/Tuples.cs

@@ -12,9 +12,9 @@ namespace Semmle.Extraction
             trapFile.WriteTuple("containerparent", parent, child);
         }
 
-        internal static void extractor_messages(this System.IO.TextWriter trapFile, ExtractionMessage error, Semmle.Util.Logging.Severity severity, string errorMessage, string entityText, Location location, string stackTrace)
+        internal static void extractor_messages(this System.IO.TextWriter trapFile, ExtractionMessage error, Semmle.Util.Logging.Severity severity, string origin, string errorMessage, string entityText, Location location, string stackTrace)
         {
-            trapFile.WriteTuple("extractor_messages", error, (int)severity, "C# extractor", errorMessage, entityText, location, stackTrace);
+            trapFile.WriteTuple("extractor_messages", error, (int)severity, origin, errorMessage, entityText, location, stackTrace);
         }
 
         public static void files(this System.IO.TextWriter trapFile, File file, string fullName)

csharp/extractor/Semmle.Util/BuildActions.cs

@@ -6,7 +6,7 @@ using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Xml;
-using Semmle.Util.Logging;
+using Semmle.Util;
 
 namespace Semmle.Util
 {
@@ -165,7 +165,7 @@ namespace Semmle.Util
         /// <summary>
         /// Downloads the resource with the specified URI to a local file.
         /// </summary>
-        void DownloadFile(string address, string fileName, ILogger logger);
+        void DownloadFile(string address, string fileName);
 
         /// <summary>
         /// Creates an <see cref="IDiagnosticsWriter" /> for the given <paramref name="filename" />.
@@ -280,8 +280,8 @@ namespace Semmle.Util
 
         public string EnvironmentExpandEnvironmentVariables(string s) => Environment.ExpandEnvironmentVariables(s);
 
-        public void DownloadFile(string address, string fileName, ILogger logger) =>
-            FileUtils.DownloadFile(address, fileName, logger);
+        public void DownloadFile(string address, string fileName) =>
+            FileUtils.DownloadFile(address, fileName);
 
         public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new DiagnosticsStream(filename);
 

csharp/extractor/Semmle.Util/BuildScript.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using Semmle.Util.Logging;
 
 namespace Semmle.Util
 {
@@ -276,14 +275,14 @@ namespace Semmle.Util
         /// <summary>
         /// Creates a build script that downloads the specified file.
         /// </summary>
-        public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback, ILogger logger) =>
+        public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback) =>
             Create(actions =>
             {
                 if (actions.GetDirectoryName(fileName) is string dir && !string.IsNullOrWhiteSpace(dir))
                     actions.CreateDirectory(dir);
                 try
                 {
-                    actions.DownloadFile(address, fileName, logger);
+                    actions.DownloadFile(address, fileName);
                     return 0;
                 }
                 catch (Exception e)

csharp/extractor/Semmle.Util/FileUtils.cs

@@ -4,7 +4,6 @@ using System.Linq;
 using System.Net.Http;
 using System.Security.Cryptography;
 using System.Text;
-using System.Threading;
 using System.Threading.Tasks;
 using Semmle.Util.Logging;
 
@@ -87,78 +86,32 @@ namespace Semmle.Util
         }
 
         /// <summary>
-        /// Computes the hash of the file at <paramref name="filePath"/>.
+        /// Computes the hash of <paramref name="filePath"/>.
         /// </summary>
         public static string ComputeFileHash(string filePath)
         {
             using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
-            var sha = SHA256.HashData(fileStream);
-            return GetHashString(sha);
-        }
-
-        /// <summary>
-        /// Computes the hash of <paramref name="input"/>.
-        /// </summary>
-        public static string ComputeHash(string input)
-        {
-            var bytes = Encoding.Unicode.GetBytes(input);
-            var sha = MD5.HashData(bytes); // MD5 to keep it shorter than SHA256
-            return GetHashString(sha).ToUpper();
-        }
-
-        private static string GetHashString(byte[] sha)
-        {
+            using var shaAlg = SHA256.Create();
+            var sha = shaAlg.ComputeHash(fileStream);
             var hex = new StringBuilder(sha.Length * 2);
             foreach (var b in sha)
-            {
                 hex.AppendFormat("{0:x2}", b);
-            }
             return hex.ToString();
         }
 
-        private static async Task DownloadFileAsync(string address, string filename, HttpClient httpClient, CancellationToken token)
+        private static async Task DownloadFileAsync(string address, string filename)
         {
-            using var contentStream = await httpClient.GetStreamAsync(address, token);
+            using var httpClient = new HttpClient();
+            using var contentStream = await httpClient.GetStreamAsync(address);
             using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
-            await contentStream.CopyToAsync(stream, token);
-        }
-
-        private static void DownloadFileWithRetry(string address, string fileName, int tryCount, int timeoutMilliSeconds, ILogger logger)
-        {
-            logger.LogDebug($"Downloading {address} to {fileName}.");
-            using HttpClient client = new();
-
-            for (var i = 0; i < tryCount; i++)
-            {
-                logger.LogDebug($"Attempt {i + 1} of {tryCount}. Timeout: {timeoutMilliSeconds} ms.");
-                using var cts = new CancellationTokenSource();
-                cts.CancelAfter(timeoutMilliSeconds);
-                try
-                {
-                    DownloadFileAsync(address, fileName, client, cts.Token).GetAwaiter().GetResult();
-                    logger.LogDebug($"Downloaded {address} to {fileName}.");
-                    return;
-                }
-                catch (Exception exc)
-                {
-                    logger.LogDebug($"Failed to download {address} to {fileName}. Exception: {exc.Message}");
-                    timeoutMilliSeconds *= 2;
-
-                    if (i == tryCount - 1)
-                    {
-                        logger.LogDebug($"Failed to download {address} to {fileName} after {tryCount} attempts.");
-                        // Rethrowing the last exception
-                        throw;
-                    }
-                }
-            }
+            await contentStream.CopyToAsync(stream);
         }
 
         /// <summary>
         /// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
         /// </summary>
-        public static void DownloadFile(string address, string fileName, ILogger logger) =>
-           DownloadFileWithRetry(address, fileName, tryCount: 3, timeoutMilliSeconds: 10000, logger);
+        public static void DownloadFile(string address, string fileName) =>
+           DownloadFileAsync(address, fileName).GetAwaiter().GetResult();
 
         public static string ConvertPathToSafeRelativePath(string path)
         {