Revert "C++: update test output"

This reverts commit d27b28d371.

.bazelrc

@@ -24,5 +24,6 @@ common --registry=file:///%workspace%/misc/bazel/registry
 common --registry=https://bcr.bazel.build
 
 common --@rules_dotnet//dotnet/settings:strict_deps=false
+common --experimental_isolated_extension_usages
 
 try-import %workspace%/local.bazelrc

.bazelrc.internal

@@ -8,3 +8,4 @@ common --registry=https://bcr.bazel.build
 # its implementation packages without providing any code itself.
 # We either can depend on internal implementation details, or turn of strict deps.
 common --@rules_dotnet//dotnet/settings:strict_deps=false
+common --experimental_isolated_extension_usages

.github/workflows/buildifier.yml

@@ -24,5 +24,5 @@ jobs:
           extra_args: >
             buildifier --all-files 2>&1 ||
             (
-              echo -e "In order to format all bazel files, please run:\n  bazel run //misc/bazel:buildifier"; exit 1
+              echo -e "In order to format all bazel files, please run:\n  bazel run //misc/bazel/buildifier"; exit 1
             )

.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
         name: Format bazel files
         files: \.(bazel|bzl)
         language: system
-        entry: bazel run //misc/bazel:buildifier
+        entry: bazel run //misc/bazel/buildifier
         pass_filenames: false
 
 #      DISABLED: can be enabled by copying this config and installing `pre-commit` with `--config` on the copy

MODULE.bazel

@@ -1,6 +1,7 @@
 module(
-    name = "codeql",
+    name = "ql",
     version = "0.0",
+    repo_name = "codeql",
 )
 
 # this points to our internal repository when `codeql` is checked out as a submodule thereof
@@ -30,11 +31,13 @@ bazel_dep(name = "rules_rust", version = "0.49.1")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
-crate = use_extension(
+# crate_py but shortened due to Windows file path considerations
+cp = use_extension(
     "@rules_rust//crate_universe:extension.bzl",
     "crate",
+    isolate = True,
 )
-crate.from_cargo(
+cp.from_cargo(
     name = "py_deps",
     cargo_lockfile = "//python/extractor/tsg-python:Cargo.lock",
     manifests = [
@@ -42,15 +45,23 @@ crate.from_cargo(
         "//python/extractor/tsg-python/tsp:Cargo.toml",
     ],
 )
-crate.from_cargo(
-    name = "ruby_deps",
+use_repo(cp, "py_deps")
+
+# crate_ruby, but shortened due to windows file paths
+r = use_extension(
+    "@rules_rust//crate_universe:extension.bzl",
+    "crate",
+    isolate = True,
+)
+r.from_cargo(
+    name = "rd",
     cargo_lockfile = "//ruby/extractor:Cargo.lock",
     manifests = [
         "//ruby/extractor:Cargo.toml",
         "//ruby/extractor/codeql-extractor-fake-crate:Cargo.toml",
     ],
 )
-use_repo(crate, "py_deps", "ruby_deps")
+use_repo(r, ruby_deps = "rd")
 
 dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet")
 dotnet.toolchain(dotnet_version = "8.0.101")

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 1.4.0
+version: 1.4.1-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/dataflow/ExternalFlow.qll

@@ -953,21 +953,3 @@ private class SummarizedCallableAdapter extends SummarizedCallable {
     interpretSummary(this, _, _, _, provenance, _)
   }
 }
-
-// adapter class for converting Mad neutrals to `NeutralCallable`s
-private class NeutralCallableAdapter extends NeutralCallable {
-  string kind;
-  string provenance_;
-
-  NeutralCallableAdapter() {
-    // Neutral models have not been implemented for CPP.
-    none() and
-    exists(this) and
-    exists(kind) and
-    exists(provenance_)
-  }
-
-  override string getKind() { result = kind }
-
-  override predicate hasProvenance(Provenance provenance) { provenance = provenance_ }
-}

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl1.qll

@@ -290,14 +290,6 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
-  predicate sourceGrouping(Node source, string sourceGroup) {
-    any(Configuration config).sourceGrouping(source, sourceGroup)
-  }
-
-  predicate sinkGrouping(Node sink, string sinkGroup) {
-    any(Configuration config).sinkGrouping(sink, sinkGroup)
-  }
-
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll

@@ -290,14 +290,6 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
-  predicate sourceGrouping(Node source, string sourceGroup) {
-    any(Configuration config).sourceGrouping(source, sourceGroup)
-  }
-
-  predicate sinkGrouping(Node sink, string sinkGroup) {
-    any(Configuration config).sinkGrouping(sink, sinkGroup)
-  }
-
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll

@@ -290,14 +290,6 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
-  predicate sourceGrouping(Node source, string sourceGroup) {
-    any(Configuration config).sourceGrouping(source, sourceGroup)
-  }
-
-  predicate sinkGrouping(Node sink, string sinkGroup) {
-    any(Configuration config).sinkGrouping(sink, sinkGroup)
-  }
-
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl4.qll

@@ -290,14 +290,6 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
-  predicate sourceGrouping(Node source, string sourceGroup) {
-    any(Configuration config).sourceGrouping(source, sourceGroup)
-  }
-
-  predicate sinkGrouping(Node sink, string sinkGroup) {
-    any(Configuration config).sinkGrouping(sink, sinkGroup)
-  }
-
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplLocal.qll

@@ -290,14 +290,6 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
-  predicate sourceGrouping(Node source, string sourceGroup) {
-    any(Configuration config).sourceGrouping(source, sourceGroup)
-  }
-
-  predicate sinkGrouping(Node sink, string sinkGroup) {
-    any(Configuration config).sinkGrouping(sink, sinkGroup)
-  }
-
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl1.qll

@@ -290,14 +290,6 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
-  predicate sourceGrouping(Node source, string sourceGroup) {
-    any(Configuration config).sourceGrouping(source, sourceGroup)
-  }
-
-  predicate sinkGrouping(Node sink, string sinkGroup) {
-    any(Configuration config).sinkGrouping(sink, sinkGroup)
-  }
-
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll

@@ -290,14 +290,6 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
-  predicate sourceGrouping(Node source, string sourceGroup) {
-    any(Configuration config).sourceGrouping(source, sourceGroup)
-  }
-
-  predicate sinkGrouping(Node sink, string sinkGroup) {
-    any(Configuration config).sinkGrouping(sink, sinkGroup)
-  }
-
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll

@@ -290,14 +290,6 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
-  predicate sourceGrouping(Node source, string sourceGroup) {
-    any(Configuration config).sourceGrouping(source, sourceGroup)
-  }
-
-  predicate sinkGrouping(Node sink, string sinkGroup) {
-    any(Configuration config).sinkGrouping(sink, sinkGroup)
-  }
-
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll

@@ -290,14 +290,6 @@ deprecated private module Config implements FullStateConfigSig {
 
   FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
 
-  predicate sourceGrouping(Node source, string sourceGroup) {
-    any(Configuration config).sourceGrouping(source, sourceGroup)
-  }
-
-  predicate sinkGrouping(Node sink, string sinkGroup) {
-    any(Configuration config).sinkGrouping(sink, sinkGroup)
-  }
-
   predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
 }
 

cpp/ql/src/Security/CWE/CWE-190/Bounded.qll

@@ -24,6 +24,10 @@ private predicate boundedBitwiseAnd(Expr e, Expr andExpr, Expr operand1, Expr op
  * operation that may greatly reduce the range of possible values.
  */
 predicate bounded(Expr e) {
+  // There can be two separate reasons for `convertedExprMightOverflow` not holding:
+  // 1. `e` really cannot overflow.
+  // 2. `e` isn't analyzable.
+  // If we didn't rule out case 2 we would declare anything that isn't analyzable as bounded.
   (
     e instanceof UnaryArithmeticOperation or
     e instanceof BinaryArithmeticOperation or

cpp/ql/src/Security/CWE/CWE-190/TaintedAllocationSize.c

@@ -1,11 +1,9 @@
 int factor = atoi(getenv("BRANCHING_FACTOR"));
 
-// GOOD: Prevent overflow by checking the input
-if (factor < 0 || factor > 1000) {
-    log("Factor out of range (%d)\n", factor);
-    return -1;
-}
-
-// This line can allocate too little memory if factor
-// is very large.
+// BAD: This can allocate too little memory if factor is very large due to overflow.
 char **root_node = (char **) malloc(factor * sizeof(char *));
+
+// GOOD: Prevent overflow and unbounded allocation size by checking the input.
+if (factor > 0 && factor <= 1000) {
+    char **root_node = (char **) malloc(factor * sizeof(char *));
+}

cpp/ql/src/Security/CWE/CWE-190/TaintedAllocationSize.qhelp

@@ -3,12 +3,16 @@
   "qhelp.dtd">
 <qhelp>
 <overview>
-<p>This code calculates an allocation size by multiplying a user input
-by a <code>sizeof</code> expression. Since the user input has no
-apparent guard on its magnitude, this multiplication can
-overflow. When an integer multiply overflows in C, the result can wrap
-around and be much smaller than intended. A later attempt to put data
-into the allocated buffer can then overflow.</p>
+
+<p>This code allocates memory using a size value based on user input,
+with no apparent bound on its magnitude being established. This allows
+for arbitrary amounts of memory to be allocated.</p>
+
+<p>If the allocation size is calculated by multiplying user input by a
+<code>sizeof</code> expression, the multiplication can overflow. When
+an integer multiplication overflows in C, the result wraps around and
+can be much smaller than intended. A later attempt to write data into
+the allocated memory can then be out of bounds.</p>
 
 </overview>
 <recommendation>

cpp/ql/src/Security/CWE/CWE-190/TaintedAllocationSize.ql

@@ -1,7 +1,7 @@
 /**
- * @name Overflow in uncontrolled allocation size
- * @description Allocating memory with a size controlled by an external
- *              user can result in integer overflow.
+ * @name Uncontrolled allocation size
+ * @description Allocating memory with a size controlled by an external user can result in
+ *              arbitrary amounts of memory being allocated.
  * @kind path-problem
  * @problem.severity error
  * @security-severity 8.1
@@ -20,6 +20,7 @@ import semmle.code.cpp.ir.IR
 import semmle.code.cpp.controlflow.IRGuards
 import semmle.code.cpp.security.FlowSources
 import TaintedAllocationSize::PathGraph
+import Bounded
 
 /**
  * Holds if `alloc` is an allocation, and `tainted` is a child of it that is a
@@ -61,16 +62,7 @@ module TaintedAllocationSizeConfig implements DataFlow::ConfigSig {
 
   predicate isBarrier(DataFlow::Node node) {
     exists(Expr e | e = node.asExpr() |
-      // There can be two separate reasons for `convertedExprMightOverflow` not holding:
-      // 1. `e` really cannot overflow.
-      // 2. `e` isn't analyzable.
-      // If we didn't rule out case 2 we would place barriers on anything that isn't analyzable.
-      (
-        e instanceof UnaryArithmeticOperation or
-        e instanceof BinaryArithmeticOperation or
-        e instanceof AssignArithmeticOperation
-      ) and
-      not convertedExprMightOverflow(e)
+      bounded(e)
       or
       // Subtracting two pointers is either well-defined (and the result will likely be small), or
       // terribly undefined and dangerous. Here, we assume that the programmer has ensured that the
@@ -104,5 +96,6 @@ where
   isFlowSource(source.getNode(), taintCause) and
   TaintedAllocationSize::flowPath(source, sink) and
   allocSink(alloc, sink.getNode())
-select alloc, source, sink, "This allocation size is derived from $@ and might overflow.",
+select alloc, source, sink,
+  "This allocation size is derived from $@ and could allocate arbitrary amounts of memory.",
   source.getNode(), "user input (" + taintCause + ")"

cpp/ql/src/change-notes/2024-08-16-uncontrolled-allocation-size.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The `cpp/uncontrolled-allocation-size` ("Uncontrolled allocation size") query now considers arithmetic operations that might reduce the size of user input as a barrier. The query therefore produces fewer false positive results.

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 1.2.0
+version: 1.2.1-dev
 groups:
   - cpp
   - queries

cpp/ql/test/library-tests/dataflow/fields/ir-path-flow.expected

@@ -848,6 +848,8 @@ edges
 | simple.cpp:120:8:120:8 | *a [i] | simple.cpp:120:10:120:10 | i | provenance |  |
 | struct_init.c:14:24:14:25 | *ab [a] | struct_init.c:14:24:14:25 | *ab [a] | provenance |  |
 | struct_init.c:14:24:14:25 | *ab [a] | struct_init.c:15:8:15:9 | *ab [a] | provenance |  |
+| struct_init.c:14:24:14:25 | *ab [a] | struct_init.c:15:8:15:9 | *ab [a] | provenance |  |
+| struct_init.c:15:8:15:9 | *ab [a] | struct_init.c:15:12:15:12 | a | provenance |  |
 | struct_init.c:15:8:15:9 | *ab [a] | struct_init.c:15:12:15:12 | a | provenance |  |
 | struct_init.c:20:13:20:14 | *definition of ab [a] | struct_init.c:22:8:22:9 | *ab [a] | provenance |  |
 | struct_init.c:20:13:20:14 | *definition of ab [a] | struct_init.c:24:10:24:12 | *& ... [a] | provenance |  |
@@ -1758,6 +1760,8 @@ nodes
 | simple.cpp:120:10:120:10 | i | semmle.label | i |
 | struct_init.c:14:24:14:25 | *ab [a] | semmle.label | *ab [a] |
 | struct_init.c:14:24:14:25 | *ab [a] | semmle.label | *ab [a] |
+| struct_init.c:14:24:14:25 | *ab [a] | semmle.label | *ab [a] |
+| struct_init.c:15:8:15:9 | *ab [a] | semmle.label | *ab [a] |
 | struct_init.c:15:8:15:9 | *ab [a] | semmle.label | *ab [a] |
 | struct_init.c:15:12:15:12 | a | semmle.label | a |
 | struct_init.c:20:13:20:14 | *definition of ab [a] | semmle.label | *definition of ab [a] |

cpp/ql/test/library-tests/dataflow/fields/path-flow.expected

@@ -737,6 +737,8 @@ edges
 | simple.cpp:120:8:120:8 | a [i] | simple.cpp:120:10:120:10 | i | provenance |  |
 | struct_init.c:14:24:14:25 | ab [a] | struct_init.c:14:24:14:25 | ab [a] | provenance |  |
 | struct_init.c:14:24:14:25 | ab [a] | struct_init.c:15:8:15:9 | ab [a] | provenance |  |
+| struct_init.c:14:24:14:25 | ab [a] | struct_init.c:15:8:15:9 | ab [a] | provenance |  |
+| struct_init.c:15:8:15:9 | ab [a] | struct_init.c:15:12:15:12 | a | provenance |  |
 | struct_init.c:15:8:15:9 | ab [a] | struct_init.c:15:12:15:12 | a | provenance |  |
 | struct_init.c:15:8:15:9 | ab [a] | struct_init.c:15:12:15:12 | a | provenance |  |
 | struct_init.c:15:8:15:9 | ab [post update] [a] | struct_init.c:14:24:14:25 | ab [a] | provenance |  |
@@ -1549,6 +1551,8 @@ nodes
 | simple.cpp:120:10:120:10 | i | semmle.label | i |
 | struct_init.c:14:24:14:25 | ab [a] | semmle.label | ab [a] |
 | struct_init.c:14:24:14:25 | ab [a] | semmle.label | ab [a] |
+| struct_init.c:14:24:14:25 | ab [a] | semmle.label | ab [a] |
+| struct_init.c:15:8:15:9 | ab [a] | semmle.label | ab [a] |
 | struct_init.c:15:8:15:9 | ab [a] | semmle.label | ab [a] |
 | struct_init.c:15:8:15:9 | ab [post update] [a] | semmle.label | ab [post update] [a] |
 | struct_init.c:15:12:15:12 | a | semmle.label | a |

cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/TaintedAllocationSize/TaintedAllocationSize.expected

@@ -13,26 +13,26 @@ edges
 | test.cpp:133:19:133:32 | *call to getenv | test.cpp:133:14:133:17 | call to atoi | provenance | TaintFunction |
 | test.cpp:148:15:148:18 | call to atol | test.cpp:152:11:152:28 | ... * ... | provenance |  |
 | test.cpp:148:20:148:33 | *call to getenv | test.cpp:148:15:148:18 | call to atol | provenance | TaintFunction |
-| test.cpp:209:8:209:23 | *get_tainted_size | test.cpp:241:9:241:24 | call to get_tainted_size | provenance |  |
-| test.cpp:211:9:211:42 | ... * ... | test.cpp:209:8:209:23 | *get_tainted_size | provenance |  |
-| test.cpp:211:14:211:27 | *call to getenv | test.cpp:211:9:211:42 | ... * ... | provenance | TaintFunction |
-| test.cpp:230:21:230:21 | s | test.cpp:231:21:231:21 | s | provenance |  |
-| test.cpp:237:19:237:52 | ... * ... | test.cpp:239:9:239:18 | local_size | provenance |  |
-| test.cpp:237:19:237:52 | ... * ... | test.cpp:245:11:245:20 | local_size | provenance |  |
-| test.cpp:237:19:237:52 | ... * ... | test.cpp:247:10:247:19 | local_size | provenance |  |
-| test.cpp:237:24:237:37 | *call to getenv | test.cpp:237:19:237:52 | ... * ... | provenance | TaintFunction |
-| test.cpp:247:10:247:19 | local_size | test.cpp:230:21:230:21 | s | provenance |  |
-| test.cpp:250:20:250:27 | *out_size | test.cpp:289:17:289:20 | get_size output argument | provenance |  |
-| test.cpp:250:20:250:27 | *out_size | test.cpp:305:18:305:21 | get_size output argument | provenance |  |
-| test.cpp:251:2:251:32 | ... = ... | test.cpp:250:20:250:27 | *out_size | provenance |  |
-| test.cpp:251:18:251:31 | *call to getenv | test.cpp:251:2:251:32 | ... = ... | provenance | TaintFunction |
-| test.cpp:259:15:259:18 | call to atoi | test.cpp:263:11:263:29 | ... * ... | provenance |  |
-| test.cpp:259:20:259:33 | *call to getenv | test.cpp:259:15:259:18 | call to atoi | provenance | TaintFunction |
-| test.cpp:289:17:289:20 | get_size output argument | test.cpp:291:11:291:28 | ... * ... | provenance |  |
-| test.cpp:305:18:305:21 | get_size output argument | test.cpp:308:10:308:27 | ... * ... | provenance |  |
-| test.cpp:353:13:353:16 | call to atoi | test.cpp:355:35:355:38 | size | provenance |  |
-| test.cpp:353:13:353:16 | call to atoi | test.cpp:356:35:356:38 | size | provenance |  |
-| test.cpp:353:18:353:31 | *call to getenv | test.cpp:353:13:353:16 | call to atoi | provenance | TaintFunction |
+| test.cpp:224:8:224:23 | *get_tainted_size | test.cpp:256:9:256:24 | call to get_tainted_size | provenance |  |
+| test.cpp:226:9:226:42 | ... * ... | test.cpp:224:8:224:23 | *get_tainted_size | provenance |  |
+| test.cpp:226:14:226:27 | *call to getenv | test.cpp:226:9:226:42 | ... * ... | provenance | TaintFunction |
+| test.cpp:245:21:245:21 | s | test.cpp:246:21:246:21 | s | provenance |  |
+| test.cpp:252:19:252:52 | ... * ... | test.cpp:254:9:254:18 | local_size | provenance |  |
+| test.cpp:252:19:252:52 | ... * ... | test.cpp:260:11:260:20 | local_size | provenance |  |
+| test.cpp:252:19:252:52 | ... * ... | test.cpp:262:10:262:19 | local_size | provenance |  |
+| test.cpp:252:24:252:37 | *call to getenv | test.cpp:252:19:252:52 | ... * ... | provenance | TaintFunction |
+| test.cpp:262:10:262:19 | local_size | test.cpp:245:21:245:21 | s | provenance |  |
+| test.cpp:265:20:265:27 | *out_size | test.cpp:304:17:304:20 | get_size output argument | provenance |  |
+| test.cpp:265:20:265:27 | *out_size | test.cpp:320:18:320:21 | get_size output argument | provenance |  |
+| test.cpp:266:2:266:32 | ... = ... | test.cpp:265:20:265:27 | *out_size | provenance |  |
+| test.cpp:266:18:266:31 | *call to getenv | test.cpp:266:2:266:32 | ... = ... | provenance | TaintFunction |
+| test.cpp:274:15:274:18 | call to atoi | test.cpp:278:11:278:29 | ... * ... | provenance |  |
+| test.cpp:274:20:274:33 | *call to getenv | test.cpp:274:15:274:18 | call to atoi | provenance | TaintFunction |
+| test.cpp:304:17:304:20 | get_size output argument | test.cpp:306:11:306:28 | ... * ... | provenance |  |
+| test.cpp:320:18:320:21 | get_size output argument | test.cpp:323:10:323:27 | ... * ... | provenance |  |
+| test.cpp:368:13:368:16 | call to atoi | test.cpp:370:35:370:38 | size | provenance |  |
+| test.cpp:368:13:368:16 | call to atoi | test.cpp:371:35:371:38 | size | provenance |  |
+| test.cpp:368:18:368:31 | *call to getenv | test.cpp:368:13:368:16 | call to atoi | provenance | TaintFunction |
 nodes
 | test.cpp:39:27:39:30 | **argv | semmle.label | **argv |
 | test.cpp:40:16:40:19 | call to atoi | semmle.label | call to atoi |
@@ -52,48 +52,48 @@ nodes
 | test.cpp:148:15:148:18 | call to atol | semmle.label | call to atol |
 | test.cpp:148:20:148:33 | *call to getenv | semmle.label | *call to getenv |
 | test.cpp:152:11:152:28 | ... * ... | semmle.label | ... * ... |
-| test.cpp:209:8:209:23 | *get_tainted_size | semmle.label | *get_tainted_size |
-| test.cpp:211:9:211:42 | ... * ... | semmle.label | ... * ... |
-| test.cpp:211:14:211:27 | *call to getenv | semmle.label | *call to getenv |
-| test.cpp:230:21:230:21 | s | semmle.label | s |
-| test.cpp:231:21:231:21 | s | semmle.label | s |
-| test.cpp:237:19:237:52 | ... * ... | semmle.label | ... * ... |
-| test.cpp:237:24:237:37 | *call to getenv | semmle.label | *call to getenv |
-| test.cpp:239:9:239:18 | local_size | semmle.label | local_size |
-| test.cpp:241:9:241:24 | call to get_tainted_size | semmle.label | call to get_tainted_size |
-| test.cpp:245:11:245:20 | local_size | semmle.label | local_size |
-| test.cpp:247:10:247:19 | local_size | semmle.label | local_size |
-| test.cpp:250:20:250:27 | *out_size | semmle.label | *out_size |
-| test.cpp:251:2:251:32 | ... = ... | semmle.label | ... = ... |
-| test.cpp:251:18:251:31 | *call to getenv | semmle.label | *call to getenv |
-| test.cpp:259:15:259:18 | call to atoi | semmle.label | call to atoi |
-| test.cpp:259:20:259:33 | *call to getenv | semmle.label | *call to getenv |
-| test.cpp:263:11:263:29 | ... * ... | semmle.label | ... * ... |
-| test.cpp:289:17:289:20 | get_size output argument | semmle.label | get_size output argument |
-| test.cpp:291:11:291:28 | ... * ... | semmle.label | ... * ... |
-| test.cpp:305:18:305:21 | get_size output argument | semmle.label | get_size output argument |
-| test.cpp:308:10:308:27 | ... * ... | semmle.label | ... * ... |
-| test.cpp:353:13:353:16 | call to atoi | semmle.label | call to atoi |
-| test.cpp:353:18:353:31 | *call to getenv | semmle.label | *call to getenv |
-| test.cpp:355:35:355:38 | size | semmle.label | size |
-| test.cpp:356:35:356:38 | size | semmle.label | size |
+| test.cpp:224:8:224:23 | *get_tainted_size | semmle.label | *get_tainted_size |
+| test.cpp:226:9:226:42 | ... * ... | semmle.label | ... * ... |
+| test.cpp:226:14:226:27 | *call to getenv | semmle.label | *call to getenv |
+| test.cpp:245:21:245:21 | s | semmle.label | s |
+| test.cpp:246:21:246:21 | s | semmle.label | s |
+| test.cpp:252:19:252:52 | ... * ... | semmle.label | ... * ... |
+| test.cpp:252:24:252:37 | *call to getenv | semmle.label | *call to getenv |
+| test.cpp:254:9:254:18 | local_size | semmle.label | local_size |
+| test.cpp:256:9:256:24 | call to get_tainted_size | semmle.label | call to get_tainted_size |
+| test.cpp:260:11:260:20 | local_size | semmle.label | local_size |
+| test.cpp:262:10:262:19 | local_size | semmle.label | local_size |
+| test.cpp:265:20:265:27 | *out_size | semmle.label | *out_size |
+| test.cpp:266:2:266:32 | ... = ... | semmle.label | ... = ... |
+| test.cpp:266:18:266:31 | *call to getenv | semmle.label | *call to getenv |
+| test.cpp:274:15:274:18 | call to atoi | semmle.label | call to atoi |
+| test.cpp:274:20:274:33 | *call to getenv | semmle.label | *call to getenv |
+| test.cpp:278:11:278:29 | ... * ... | semmle.label | ... * ... |
+| test.cpp:304:17:304:20 | get_size output argument | semmle.label | get_size output argument |
+| test.cpp:306:11:306:28 | ... * ... | semmle.label | ... * ... |
+| test.cpp:320:18:320:21 | get_size output argument | semmle.label | get_size output argument |
+| test.cpp:323:10:323:27 | ... * ... | semmle.label | ... * ... |
+| test.cpp:368:13:368:16 | call to atoi | semmle.label | call to atoi |
+| test.cpp:368:18:368:31 | *call to getenv | semmle.label | *call to getenv |
+| test.cpp:370:35:370:38 | size | semmle.label | size |
+| test.cpp:371:35:371:38 | size | semmle.label | size |
 subpaths
 #select
-| test.cpp:43:31:43:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:43:38:43:44 | tainted | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:44:31:44:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:44:38:44:63 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:46:31:46:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:46:38:46:63 | ... + ... | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:49:25:49:30 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:49:32:49:35 | size | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:50:17:50:30 | new[] | test.cpp:39:27:39:30 | **argv | test.cpp:50:17:50:30 | size | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:53:21:53:27 | call to realloc | test.cpp:39:27:39:30 | **argv | test.cpp:53:35:53:60 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:128:17:128:22 | call to malloc | test.cpp:124:18:124:31 | *call to getenv | test.cpp:128:24:128:41 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:124:18:124:31 | *call to getenv | user input (an environment variable) |
-| test.cpp:135:3:135:8 | call to malloc | test.cpp:133:19:133:32 | *call to getenv | test.cpp:135:10:135:27 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:133:19:133:32 | *call to getenv | user input (an environment variable) |
-| test.cpp:152:4:152:9 | call to malloc | test.cpp:148:20:148:33 | *call to getenv | test.cpp:152:11:152:28 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:148:20:148:33 | *call to getenv | user input (an environment variable) |
-| test.cpp:231:14:231:19 | call to malloc | test.cpp:237:24:237:37 | *call to getenv | test.cpp:231:21:231:21 | s | This allocation size is derived from $@ and might overflow. | test.cpp:237:24:237:37 | *call to getenv | user input (an environment variable) |
-| test.cpp:239:2:239:7 | call to malloc | test.cpp:237:24:237:37 | *call to getenv | test.cpp:239:9:239:18 | local_size | This allocation size is derived from $@ and might overflow. | test.cpp:237:24:237:37 | *call to getenv | user input (an environment variable) |
-| test.cpp:241:2:241:7 | call to malloc | test.cpp:211:14:211:27 | *call to getenv | test.cpp:241:9:241:24 | call to get_tainted_size | This allocation size is derived from $@ and might overflow. | test.cpp:211:14:211:27 | *call to getenv | user input (an environment variable) |
-| test.cpp:245:2:245:9 | call to my_alloc | test.cpp:237:24:237:37 | *call to getenv | test.cpp:245:11:245:20 | local_size | This allocation size is derived from $@ and might overflow. | test.cpp:237:24:237:37 | *call to getenv | user input (an environment variable) |
-| test.cpp:263:4:263:9 | call to malloc | test.cpp:259:20:259:33 | *call to getenv | test.cpp:263:11:263:29 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:259:20:259:33 | *call to getenv | user input (an environment variable) |
-| test.cpp:291:4:291:9 | call to malloc | test.cpp:251:18:251:31 | *call to getenv | test.cpp:291:11:291:28 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:251:18:251:31 | *call to getenv | user input (an environment variable) |
-| test.cpp:308:3:308:8 | call to malloc | test.cpp:251:18:251:31 | *call to getenv | test.cpp:308:10:308:27 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:251:18:251:31 | *call to getenv | user input (an environment variable) |
-| test.cpp:355:25:355:33 | call to MyMalloc1 | test.cpp:353:18:353:31 | *call to getenv | test.cpp:355:35:355:38 | size | This allocation size is derived from $@ and might overflow. | test.cpp:353:18:353:31 | *call to getenv | user input (an environment variable) |
-| test.cpp:356:25:356:33 | call to MyMalloc2 | test.cpp:353:18:353:31 | *call to getenv | test.cpp:356:35:356:38 | size | This allocation size is derived from $@ and might overflow. | test.cpp:353:18:353:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:43:31:43:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:43:38:43:44 | tainted | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:44:31:44:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:44:38:44:63 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:46:31:46:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:46:38:46:63 | ... + ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:49:25:49:30 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:49:32:49:35 | size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:50:17:50:30 | new[] | test.cpp:39:27:39:30 | **argv | test.cpp:50:17:50:30 | size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:53:21:53:27 | call to realloc | test.cpp:39:27:39:30 | **argv | test.cpp:53:35:53:60 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:128:17:128:22 | call to malloc | test.cpp:124:18:124:31 | *call to getenv | test.cpp:128:24:128:41 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:124:18:124:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:135:3:135:8 | call to malloc | test.cpp:133:19:133:32 | *call to getenv | test.cpp:135:10:135:27 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:133:19:133:32 | *call to getenv | user input (an environment variable) |
+| test.cpp:152:4:152:9 | call to malloc | test.cpp:148:20:148:33 | *call to getenv | test.cpp:152:11:152:28 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:148:20:148:33 | *call to getenv | user input (an environment variable) |
+| test.cpp:246:14:246:19 | call to malloc | test.cpp:252:24:252:37 | *call to getenv | test.cpp:246:21:246:21 | s | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:252:24:252:37 | *call to getenv | user input (an environment variable) |
+| test.cpp:254:2:254:7 | call to malloc | test.cpp:252:24:252:37 | *call to getenv | test.cpp:254:9:254:18 | local_size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:252:24:252:37 | *call to getenv | user input (an environment variable) |
+| test.cpp:256:2:256:7 | call to malloc | test.cpp:226:14:226:27 | *call to getenv | test.cpp:256:9:256:24 | call to get_tainted_size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:226:14:226:27 | *call to getenv | user input (an environment variable) |
+| test.cpp:260:2:260:9 | call to my_alloc | test.cpp:252:24:252:37 | *call to getenv | test.cpp:260:11:260:20 | local_size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:252:24:252:37 | *call to getenv | user input (an environment variable) |
+| test.cpp:278:4:278:9 | call to malloc | test.cpp:274:20:274:33 | *call to getenv | test.cpp:278:11:278:29 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:274:20:274:33 | *call to getenv | user input (an environment variable) |
+| test.cpp:306:4:306:9 | call to malloc | test.cpp:266:18:266:31 | *call to getenv | test.cpp:306:11:306:28 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:266:18:266:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:323:3:323:8 | call to malloc | test.cpp:266:18:266:31 | *call to getenv | test.cpp:323:10:323:27 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:266:18:266:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:370:25:370:33 | call to MyMalloc1 | test.cpp:368:18:368:31 | *call to getenv | test.cpp:370:35:370:38 | size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:368:18:368:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:371:25:371:33 | call to MyMalloc2 | test.cpp:368:18:368:31 | *call to getenv | test.cpp:371:35:371:38 | size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:368:18:368:31 | *call to getenv | user input (an environment variable) |

cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/TaintedAllocationSize/test.cpp

@@ -40,10 +40,10 @@ int main(int argc, char **argv) {
 	int tainted = atoi(argv[1]);
 
 	MyStruct *arr1 = (MyStruct *)malloc(sizeof(MyStruct)); // GOOD
-	MyStruct *arr2 = (MyStruct *)malloc(tainted); // DUBIOUS (not multiplied by anything)
+	MyStruct *arr2 = (MyStruct *)malloc(tainted); // BAD
 	MyStruct *arr3 = (MyStruct *)malloc(tainted * sizeof(MyStruct)); // BAD
 	MyStruct *arr4 = (MyStruct *)malloc(getTainted() * sizeof(MyStruct)); // BAD [NOT DETECTED]
-	MyStruct *arr5 = (MyStruct *)malloc(sizeof(MyStruct) + tainted); // DUBIOUS (not multiplied by anything)
+	MyStruct *arr5 = (MyStruct *)malloc(sizeof(MyStruct) + tainted); // BAD
 
 	int size = tainted * 8;
 	char *chars1 = (char *)malloc(size); // BAD
@@ -180,6 +180,21 @@ void more_bounded_tests() {
 		}
 	}
 
+	{
+		int size = atoi(getenv("USER"));
+		int size2 = size % 100;
+		malloc(size2 * sizeof(int)); // GOOD
+	}
+
+	{
+		int size = atoi(getenv("USER"));
+
+		if (size % 100)
+		{
+			malloc(size * sizeof(int)); // BAD [NOT DETECTED]
+		}
+	}
+
 	{
 		int size = atoi(getenv("USER"));
 

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -7,6 +7,7 @@ using System.Xml;
 using Microsoft.Build.Construction;
 using Semmle.Util;
 using Semmle.Autobuild.Shared;
+using Semmle.Util.Logging;
 
 namespace Semmle.Autobuild.CSharp.Tests
 {
@@ -203,7 +204,7 @@ namespace Semmle.Autobuild.CSharp.Tests
                 throw new ArgumentException($"Missing CreateDirectory, {path}");
         }
 
-        public void DownloadFile(string address, string fileName)
+        public void DownloadFile(string address, string fileName, ILogger logger)
         {
             if (!DownloadFiles.Contains((address, fileName)))
                 throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");

csharp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -7,6 +7,7 @@ using System.Linq;
 using Microsoft.Build.Construction;
 using System.Xml;
 using System.IO;
+using Semmle.Util.Logging;
 
 namespace Semmle.Autobuild.Cpp.Tests
 {
@@ -189,7 +190,7 @@ namespace Semmle.Autobuild.Cpp.Tests
                 throw new ArgumentException($"Missing CreateDirectory, {path}");
         }
 
-        public void DownloadFile(string address, string fileName)
+        public void DownloadFile(string address, string fileName, ILogger logger)
         {
             if (!DownloadFiles.Contains((address, fileName)))
                 throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");

csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs

@@ -157,7 +157,8 @@ namespace Semmle.Autobuild.Shared
             BuildScript.DownloadFile(
                 FileUtils.NugetExeUrl,
                 path,
-                e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}"))
+                e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}"),
+                builder.Logger)
             &
             BuildScript.Create(_ =>
             {

csharp/documentation/library-coverage/coverage.csv

@@ -1,5 +1,5 @@
-package,sink,source,summary,sink:code-injection,sink:encryption-decryptor,sink:encryption-encryptor,sink:encryption-keyprop,sink:encryption-symmetrickey,sink:file-content-store,sink:html-injection,sink:js-injection,sink:log-injection,sink:sql-injection,source:commandargs,source:database,source:environment,source:file,source:file-write,source:local,source:remote,source:windows-registry,summary:taint,summary:value
-Amazon.Lambda.APIGatewayEvents,,6,,,,,,,,,,,,,,,,,,6,,,
+package,sink,source,summary,sink:code-injection,sink:encryption-decryptor,sink:encryption-encryptor,sink:encryption-keyprop,sink:encryption-symmetrickey,sink:file-content-store,sink:html-injection,sink:js-injection,sink:log-injection,sink:sql-injection,source:commandargs,source:database,source:environment,source:file,source:file-write,source:remote,source:stdin,source:windows-registry,summary:taint,summary:value
+Amazon.Lambda.APIGatewayEvents,,6,,,,,,,,,,,,,,,,,6,,,,
 Amazon.Lambda.Core,10,,,,,,,,,,,10,,,,,,,,,,,
 Dapper,55,42,1,,,,,,,,,,55,,42,,,,,,,,1
 ILCompiler,,,123,,,,,,,,,,,,,,,,,,,123,
@@ -10,7 +10,7 @@ Internal.IL,,,46,,,,,,,,,,,,,,,,,,,44,2
 Internal.Pgo,,,9,,,,,,,,,,,,,,,,,,,8,1
 Internal.TypeSystem,,,315,,,,,,,,,,,,,,,,,,,299,16
 JsonToItemsTaskFactory,,,10,,,,,,,,,,,,,,,,,,,10,
-Microsoft.Android.Build,,,16,,,,,,,,,,,,,,,,,,,16,
+Microsoft.Android.Build,,1,16,,,,,,,,,,,,,1,,,,,,16,
 Microsoft.Apple.Build,,,8,,,,,,,,,,,,,,,,,,,8,
 Microsoft.ApplicationBlocks.Data,28,,,,,,,,,,,,28,,,,,,,,,,
 Microsoft.CSharp,,,13,,,,,,,,,,,,,,,,,,,13,
@@ -19,9 +19,9 @@ Microsoft.DotNet.Build.Tasks,,,6,,,,,,,,,,,,,,,,,,,6,
 Microsoft.EntityFrameworkCore,6,,12,,,,,,,,,,6,,,,,,,,,,12
 Microsoft.Extensions.Caching.Distributed,,,10,,,,,,,,,,,,,,,,,,,10,
 Microsoft.Extensions.Caching.Memory,,,39,,,,,,,,,,,,,,,,,,,38,1
-Microsoft.Extensions.Configuration,,2,90,,,,,,,,,,,,,2,,,,,,89,1
+Microsoft.Extensions.Configuration,,3,90,,,,,,,,,,,,,3,,,,,,89,1
 Microsoft.Extensions.DependencyInjection,,,134,,,,,,,,,,,,,,,,,,,133,1
-Microsoft.Extensions.DependencyModel,,,18,,,,,,,,,,,,,,,,,,,18,
+Microsoft.Extensions.DependencyModel,,1,18,,,,,,,,,,,,,1,,,,,,18,
 Microsoft.Extensions.Diagnostics.Metrics,,,15,,,,,,,,,,,,,,,,,,,15,
 Microsoft.Extensions.FileProviders,,,15,,,,,,,,,,,,,,,,,,,15,
 Microsoft.Extensions.FileSystemGlobbing,,,18,,,,,,,,,,,,,,,,,,,16,2
@@ -41,5 +41,5 @@ MySql.Data.MySqlClient,48,,,,,,,,,,,,48,,,,,,,,,,
 Newtonsoft.Json,,,91,,,,,,,,,,,,,,,,,,,73,18
 ServiceStack,194,,7,27,,,,,75,,,,92,,,,,,,,,7,
 SourceGenerators,,,5,,,,,,,,,,,,,,,,,,,5,
-System,60,44,10614,,7,6,5,,,4,5,,33,2,,3,15,17,3,4,,8709,1905
+System,54,47,10626,,6,5,5,,,4,1,,33,2,,6,15,17,4,3,,8721,1905
 Windows.Security.Cryptography.Core,1,,,,,,,1,,,,,,,,,,,,,,,

csharp/documentation/library-coverage/coverage.rst

@@ -8,7 +8,7 @@ C# framework & library support
 
    Framework / library,Package,Flow sources,Taint & value steps,Sinks (total),`CWE-079` :sub:`Cross-site scripting`
    `ServiceStack <https://servicestack.net/>`_,"``ServiceStack.*``, ``ServiceStack``",,7,194,
-   System,"``System.*``, ``System``",44,10614,60,9
-   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.DotNet.Build.Tasks``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",54,1821,148,
-   Totals,,98,12442,402,9
+   System,"``System.*``, ``System``",47,10626,54,5
+   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.DotNet.Build.Tasks``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",57,1821,148,
+   Totals,,104,12454,396,5
 

csharp/downgrades/15b989afd2bfc4743536fdb0958c1d8177a32600/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add unique constraint on preprocessor directive and compilation pairs
+compatibility: backwards

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs

@@ -189,13 +189,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// (together with used package information) required for compilation.
         /// </summary>
         /// <returns>True if parsing succeeds, otherwise false.</returns>
-        public bool TryParse(string json)
+        public bool TryParse(string json, string jsonPath)
         {
             try
             {
                 var obj = JObject.Parse(json);
-                AddPackageDependencies(obj, json);
-                AddFrameworkDependencies(obj, json);
+                AddPackageDependencies(obj, jsonPath);
+                AddFrameworkDependencies(obj, jsonPath);
                 return true;
             }
             catch (Exception e)
@@ -228,7 +228,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         {
             if (TryReadAllText(asset, logger, out var json))
             {
-                TryParse(json);
+                TryParse(json, asset);
             }
         }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -248,7 +248,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                     var downloadDotNetInstallSh = BuildScript.DownloadFile(
                         "https://dot.net/v1/dotnet-install.sh",
                         dotnetInstallPath,
-                        e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"));
+                        e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"),
+                        logger);
 
                     var chmod = new CommandBuilder(actions).
                         RunCommand("chmod").

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs

@@ -145,7 +145,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
             Directory.CreateDirectory(directory);
             logger.LogInfo("Attempting to download nuget.exe");
-            FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget);
+            FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget, logger);
             logger.LogInfo($"Downloaded nuget.exe to {nuget}");
             return nuget;
         }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -44,9 +44,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             this.logger = logger;
             this.compilationInfoContainer = compilationInfoContainer;
 
-            PackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath(fileProvider.SourceDir.FullName, "packages"), "package", logger);
-            legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath(fileProvider.SourceDir.FullName, "legacypackages"), "legacy package", logger);
-            missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath(fileProvider.SourceDir.FullName, "missingpackages"), "missing package", logger);
+            PackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("packages"), "package", logger);
+            legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("legacypackages"), "legacy package", logger);
+            missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("missingpackages"), "missing package", logger);
         }
 
         public string? TryRestore(string package)
@@ -338,7 +338,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
 
             logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
-            using var tempDir = new TemporaryDirectory(ComputeTempDirectoryPath(fileProvider.SourceDir.FullName, "nugetconfig"), "generated nuget config", logger);
+            using var tempDir = new TemporaryDirectory(ComputeTempDirectoryPath("nugetconfig"), "generated nuget config", logger);
             var nugetConfig = fallbackNugetFeeds is null
                 ? GetNugetConfig()
                 : CreateFallbackNugetConfig(fallbackNugetFeeds, tempDir.DirInfo.FullName);
@@ -667,7 +667,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                     "Found unreachable Nuget feed in C# analysis with build-mode 'none'",
                     visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
                     markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
-                    severity: DiagnosticMessage.TspSeverity.Warning
+                    severity: DiagnosticMessage.TspSeverity.Note
                 ));
             }
             compilationInfoContainer.CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));
@@ -771,19 +771,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         }
 
         /// <summary>
-        /// Computes a unique temp directory for the packages associated
-        /// with this source tree. Use a SHA1 of the directory name.
+        /// Returns the full path to a temporary directory with the given subfolder name.
+        /// </summary>
+        private static string ComputeTempDirectoryPath(string subfolderName)
+        {
+            return Path.Combine(FileUtils.GetTemporaryWorkingDirectory(out _), subfolderName);
+        }
+
+        /// <summary>
+        /// Computes a unique temporary directory path based on the source directory and the subfolder name.
         /// </summary>
-        /// <returns>The full path of the temp directory.</returns>
         private static string ComputeTempDirectoryPath(string srcDir, string subfolderName)
         {
-            var bytes = Encoding.Unicode.GetBytes(srcDir);
-            var sha = SHA1.HashData(bytes);
-            var sb = new StringBuilder();
-            foreach (var b in sha.Take(8))
-                sb.AppendFormat("{0:x2}", b);
-
-            return Path.Combine(FileUtils.GetTemporaryWorkingDirectory(out _), sb.ToString(), subfolderName);
+            return Path.Combine(FileUtils.GetTemporaryWorkingDirectory(out _), FileUtils.ComputeHash(srcDir), subfolderName);
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs

@@ -83,7 +83,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 var targetDir = GetTemporaryWorkingDirectory(FileType.ToLowerInvariant());
 
                 return groupedFiles
-                    .SelectMany(group => sourceGenerator.RunSourceGenerator(group.Value, group.Key, references, targetDir));
+                    .SelectMany(group => sourceGenerator.RunSourceGenerator(group.Value, group.Key, references, targetDir, fileProvider.SourceDir.FullName));
             }
             catch (Exception ex)
             {

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorWrapper/DotnetSourceGeneratorWrapper.cs

@@ -33,11 +33,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         protected abstract void GenerateAnalyzerConfig(IEnumerable<string> additionalFiles, string csprojFile, string analyzerConfigPath);
 
-        public IEnumerable<string> RunSourceGenerator(IEnumerable<string> additionalFiles, string csprojFile, IEnumerable<string> references, string targetDir)
+        public IEnumerable<string> RunSourceGenerator(IEnumerable<string> additionalFiles, string csprojFile, IEnumerable<string> references, string targetDir, string sourceDir)
         {
             try
             {
-                var name = Guid.NewGuid().ToString("N").ToUpper();
+                var relativePathToCsProj = Path.GetRelativePath(sourceDir, csprojFile);
+                var name = FileUtils.ComputeHash($"{relativePathToCsProj}\n{this.GetType().Name}");
                 using var tempDir = new TemporaryDirectory(Path.Join(FileUtils.GetTemporaryWorkingDirectory(out _), "source-generator"), "source generator temporary", logger);
                 var analyzerConfigPath = Path.Combine(tempDir.DirInfo.FullName, $"{name}.txt");
                 var dllPath = Path.Combine(tempDir.DirInfo.FullName, $"{name}.dll");

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs

@@ -3,6 +3,7 @@ using System.Collections.Concurrent;
 using System.IO;
 using System.Linq;
 using Microsoft.CodeAnalysis;
+using Semmle.Extraction.Entities;
 using Semmle.Util;
 
 namespace Semmle.Extraction.CSharp.Entities
@@ -89,13 +90,21 @@ namespace Semmle.Extraction.CSharp.Entities
             trapFile.compilation_finished(this, (float)p.Total.Cpu.TotalSeconds, (float)p.Total.Elapsed.TotalSeconds);
         }
 
+        public void PopulateAggregatedMessages()
+        {
+            ExtractionMessage.groupedMessageCounts.ForEach(pair =>
+            {
+                Context.TrapWriter.Writer.compilation_info(this, $"Extractor message count for group '{pair.Key}'", pair.Value.ToString());
+            });
+        }
+
         public override void WriteId(EscapingTextWriter trapFile)
         {
             trapFile.Write(hashCode);
             trapFile.Write(";compilation");
         }
 
-        public override Location ReportingLocation => throw new NotImplementedException();
+        public override Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException();
 
         public override bool NeedsPopulation => Context.IsAssemblyScope;
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs

@@ -21,7 +21,6 @@ namespace Semmle.Extraction.CSharp.Entities
 
         protected override void Populate(TextWriter trapFile)
         {
-            // The below doesn't limit the extractor messages to the exact limit, but it's good enough.
             var key = diagnostic.Id;
             var messageCount = compilation.messageCounts.AddOrUpdate(key, 1, (_, c) => c + 1);
             if (messageCount > limit)

csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs

@@ -21,7 +21,7 @@ namespace Semmle.Extraction.CSharp.Entities
         public override Microsoft.CodeAnalysis.Location ReportingLocation =>
             IsCompilerGeneratedDelegate()
                 ? Symbol.ContainingType.GetSymbolLocation()
-                : Symbol.GetSymbolLocation();
+                : BodyDeclaringSymbol.GetSymbolLocation();
 
         public override bool NeedsPopulation => base.NeedsPopulation || IsCompilerGeneratedDelegate();
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/ElifDirective.cs

@@ -20,6 +20,7 @@ namespace Semmle.Extraction.CSharp.Entities
         public override void WriteId(EscapingTextWriter trapFile)
         {
             trapFile.WriteSubId(Context.CreateLocation(ReportingLocation));
+            trapFile.WriteSubId(start);
             trapFile.Write(Symbol.IsActive);
             trapFile.Write(',');
             trapFile.Write(Symbol.BranchTaken);

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/ElseDirective.cs

@@ -18,6 +18,7 @@ namespace Semmle.Extraction.CSharp.Entities
         public override void WriteId(EscapingTextWriter trapFile)
         {
             trapFile.WriteSubId(Context.CreateLocation(ReportingLocation));
+            trapFile.WriteSubId(start);
             trapFile.Write(Symbol.IsActive);
             trapFile.Write(',');
             trapFile.Write(Symbol.BranchTaken);

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/EndIfDirective.cs

@@ -13,6 +13,14 @@ namespace Semmle.Extraction.CSharp.Entities
             this.start = start;
         }
 
+        public override void WriteId(EscapingTextWriter trapFile)
+        {
+            trapFile.WriteSubId(Context.CreateLocation(ReportingLocation));
+            trapFile.WriteSubId(start);
+            trapFile.Write(Symbol.IsActive);
+            trapFile.Write(";trivia");
+        }
+
         protected override void PopulatePreprocessor(TextWriter trapFile)
         {
             trapFile.directive_endifs(this, start);

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -190,37 +190,29 @@ namespace Semmle.Extraction.CSharp
                 var transformedSourcePath = PathTransformer.Transform(sourcePath);
 
                 var trapPath = transformedSourcePath.GetTrapPath(Logger, options.TrapCompression);
-                var upToDate = false;
-
-                // compilation.Clone() is used to allow symbols to be garbage collected.
                 using var trapWriter = transformedSourcePath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
 
-                upToDate = FileIsUpToDate(sourcePath, trapWriter.TrapFile);
-
                 var currentTaskId = IncrementTaskCount();
                 ReportProgressTaskStarted(currentTaskId, sourcePath);
 
-                if (!upToDate)
+                var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
+                // Ensure that the file itself is populated in case the source file is totally empty
+                var root = tree.GetRoot();
+                Entities.File.Create(cx, root.SyntaxTree.FilePath);
+
+                var csNode = (CSharpSyntaxNode)root;
+                var directiveVisitor = new DirectiveVisitor(cx);
+                csNode.Accept(directiveVisitor);
+                foreach (var branch in directiveVisitor.BranchesTaken)
                 {
-                    var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
-                    // Ensure that the file itself is populated in case the source file is totally empty
-                    var root = tree.GetRoot();
-                    Entities.File.Create(cx, root.SyntaxTree.FilePath);
-
-                    var csNode = (CSharpSyntaxNode)root;
-                    var directiveVisitor = new DirectiveVisitor(cx);
-                    csNode.Accept(directiveVisitor);
-                    foreach (var branch in directiveVisitor.BranchesTaken)
-                    {
-                        cx.TrapStackSuffix.Add(branch);
-                    }
-                    csNode.Accept(new CompilationUnitVisitor(cx));
-                    cx.PopulateAll();
-                    CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
-                    cx.PopulateAll();
+                    cx.TrapStackSuffix.Add(branch);
                 }
+                csNode.Accept(new CompilationUnitVisitor(cx));
+                cx.PopulateAll();
+                CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
+                cx.PopulateAll();
 
-                ReportProgressTaskDone(currentTaskId, sourcePath, trapPath, stopwatch.Elapsed, upToDate ? AnalysisAction.UpToDate : AnalysisAction.Extracted);
+                ReportProgressTaskDone(currentTaskId, sourcePath, trapPath, stopwatch.Elapsed, AnalysisAction.Extracted);
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
@@ -258,6 +250,8 @@ namespace Semmle.Extraction.CSharp
 
         public void LogPerformance(Entities.PerformanceMetrics p) => compilationEntity.PopulatePerformance(p);
 
+        public void ExtractAggregatedMessages() => compilationEntity.PopulateAggregatedMessages();
+
 #nullable restore warnings
 
         /// <summary>
@@ -268,12 +262,6 @@ namespace Semmle.Extraction.CSharp
             extractionTasks.Add(() => DoAnalyseCompilation());
         }
 
-        private static bool FileIsUpToDate(string src, string dest)
-        {
-            return File.Exists(dest) &&
-                File.GetLastWriteTime(dest) >= File.GetLastWriteTime(src);
-        }
-
         private static void AnalyseNamespace(Context cx, INamespaceSymbol ns)
         {
             foreach (var memberNamespace in ns.GetNamespaceMembers())

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -458,6 +458,7 @@ namespace Semmle.Extraction.CSharp
 
             sw.Restart();
             analyser.PerformExtraction(options.Threads);
+            analyser.ExtractAggregatedMessages();
             sw.Stop();
             var cpuTime2 = currentProcess.TotalProcessorTime;
             var userTime2 = currentProcess.UserProcessorTime;

csharp/extractor/Semmle.Extraction.Tests/Assets.cs

@@ -16,7 +16,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsJson1;
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.True(success);
@@ -46,7 +46,7 @@ namespace Semmle.Extraction.Tests
             var json = "garbage data";
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.False(success);
@@ -61,7 +61,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNet70;
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.True(success);
@@ -91,7 +91,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNet48;
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.True(success);
@@ -117,7 +117,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNetstandard21;
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.True(success);
@@ -144,7 +144,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNetstandard16;
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.True(success);
@@ -175,7 +175,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNetcoreapp20;
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.True(success);
@@ -202,7 +202,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNetcoreapp31;
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.True(success);

csharp/extractor/Semmle.Extraction/BUILD.bazel

@@ -26,6 +26,7 @@ codeql_csharp_library(
         ],
         "//conditions:default": [],
     }),
+    internals_visible_to = ["Semmle.Extraction.CSharp"],
     visibility = ["//csharp:__subpackages__"],
     deps = [
         "//csharp/extractor/Semmle.Util",

csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs

@@ -1,4 +1,5 @@
-using System.IO;
+using System.Collections.Concurrent;
+using System.IO;
 using System.Threading;
 using Semmle.Util;
 
@@ -7,30 +8,45 @@ namespace Semmle.Extraction.Entities
     internal class ExtractionMessage : FreshEntity
     {
         private static readonly int limit = EnvironmentVariables.TryGetExtractorNumberOption<int>("MESSAGE_LIMIT") ?? 10000;
+
+        internal static readonly ConcurrentDictionary<string, int> groupedMessageCounts = [];
         private static int messageCount = 0;
 
         private readonly Message msg;
+        private readonly bool bypassLimit;
 
-        public ExtractionMessage(Context cx, Message msg) : base(cx)
+        public ExtractionMessage(Context cx, Message msg) : this(cx, msg, bypassLimit: false)
         {
+        }
+
+        private ExtractionMessage(Context cx, Message msg, bool bypassLimit) : base(cx)
+        {
+            this.bypassLimit = bypassLimit;
             this.msg = msg;
             TryPopulate();
         }
 
         protected override void Populate(TextWriter trapFile)
         {
-            // The below doesn't limit the extractor messages to the exact limit, but it's good enough.
-            Interlocked.Increment(ref messageCount);
-            if (messageCount > limit)
+            // For the time being we're counting the number of messages per severity, we could introduce other groupings in the future
+            var key = msg.Severity.ToString();
+            groupedMessageCounts.AddOrUpdate(key, 1, (_, c) => c + 1);
+
+            if (!bypassLimit)
             {
-                if (messageCount == limit + 1)
+                var val = Interlocked.Increment(ref messageCount);
+                if (val > limit)
                 {
-                    Context.ExtractionContext.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}");
+                    if (val == limit + 1)
+                    {
+                        Context.ExtractionContext.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}");
+                        _ = new ExtractionMessage(Context, new Message($"Stopped logging extractor messages after reaching {limit}", null, null, null, Util.Logging.Severity.Warning), bypassLimit: true);
+                    }
+                    return;
                 }
-                return;
             }
 
-            trapFile.extractor_messages(this, msg.Severity, "C# extractor", msg.Text, msg.EntityText ?? string.Empty,
+            trapFile.extractor_messages(this, msg.Severity, msg.Text, msg.EntityText ?? string.Empty,
                 msg.Location ?? Context.CreateLocation(), msg.StackTrace ?? string.Empty);
         }
     }

csharp/extractor/Semmle.Extraction/Semmle.Extraction.csproj

@@ -5,6 +5,8 @@
 	</PropertyGroup>
 	<ItemGroup>
 		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
+
+		<InternalsVisibleTo Include="Semmle.Extraction.CSharp" />
 	</ItemGroup>
   <Import Project="..\..\.paket\Paket.Restore.targets" />
 </Project>

csharp/extractor/Semmle.Extraction/TrapWriter.cs

@@ -194,8 +194,11 @@ namespace Semmle.Extraction
                     var hash = FileUtils.ComputeFileHash(tmpFile);
                     if (existingHash != hash)
                     {
-                        var root = TrapFile.Substring(0, TrapFile.Length - 8); // Remove trailing ".trap.gz"
-                        if (TryMove(tmpFile, $"{root}-{hash}.trap{TrapExtension(trapCompression)}"))
+                        var extension = TrapExtension(trapCompression);
+                        var root = TrapFile[..^extension.Length]; // Remove trailing ".trap", ".trap.gz", or ".trap.br"
+                        var newTrapName = $"{root}-{hash}{extension}";
+                        logger.LogInfo($"Identical trap file for {TrapFile} already exists, renaming to {newTrapName}");
+                        if (TryMove(tmpFile, $"{newTrapName}"))
                             return;
                     }
                     logger.LogInfo($"Identical trap file for {TrapFile} already exists");
@@ -217,16 +220,16 @@ namespace Semmle.Extraction
         {
             switch (compression)
             {
-                case CompressionMode.None: return "";
-                case CompressionMode.Gzip: return ".gz";
-                case CompressionMode.Brotli: return ".br";
+                case CompressionMode.None: return ".trap";
+                case CompressionMode.Gzip: return ".trap.gz";
+                case CompressionMode.Brotli: return ".trap.br";
                 default: throw new ArgumentOutOfRangeException(nameof(compression), compression, "Unsupported compression type");
             }
         }
 
         public static string TrapPath(ILogger logger, string? folder, PathTransformer.ITransformedPath path, TrapWriter.CompressionMode trapCompression)
         {
-            var filename = $"{path.Value}.trap{TrapExtension(trapCompression)}";
+            var filename = $"{path.Value}{TrapExtension(trapCompression)}";
             if (string.IsNullOrEmpty(folder))
                 folder = Directory.GetCurrentDirectory();
 

csharp/extractor/Semmle.Extraction/Tuples.cs

@@ -12,9 +12,9 @@ namespace Semmle.Extraction
             trapFile.WriteTuple("containerparent", parent, child);
         }
 
-        internal static void extractor_messages(this System.IO.TextWriter trapFile, ExtractionMessage error, Semmle.Util.Logging.Severity severity, string origin, string errorMessage, string entityText, Location location, string stackTrace)
+        internal static void extractor_messages(this System.IO.TextWriter trapFile, ExtractionMessage error, Semmle.Util.Logging.Severity severity, string errorMessage, string entityText, Location location, string stackTrace)
         {
-            trapFile.WriteTuple("extractor_messages", error, (int)severity, origin, errorMessage, entityText, location, stackTrace);
+            trapFile.WriteTuple("extractor_messages", error, (int)severity, "C# extractor", errorMessage, entityText, location, stackTrace);
         }
 
         public static void files(this System.IO.TextWriter trapFile, File file, string fullName)

csharp/extractor/Semmle.Util/BuildActions.cs

@@ -6,7 +6,7 @@ using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Xml;
-using Semmle.Util;
+using Semmle.Util.Logging;
 
 namespace Semmle.Util
 {
@@ -165,7 +165,7 @@ namespace Semmle.Util
         /// <summary>
         /// Downloads the resource with the specified URI to a local file.
         /// </summary>
-        void DownloadFile(string address, string fileName);
+        void DownloadFile(string address, string fileName, ILogger logger);
 
         /// <summary>
         /// Creates an <see cref="IDiagnosticsWriter" /> for the given <paramref name="filename" />.
@@ -280,8 +280,8 @@ namespace Semmle.Util
 
         public string EnvironmentExpandEnvironmentVariables(string s) => Environment.ExpandEnvironmentVariables(s);
 
-        public void DownloadFile(string address, string fileName) =>
-            FileUtils.DownloadFile(address, fileName);
+        public void DownloadFile(string address, string fileName, ILogger logger) =>
+            FileUtils.DownloadFile(address, fileName, logger);
 
         public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new DiagnosticsStream(filename);
 

csharp/extractor/Semmle.Util/BuildScript.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using Semmle.Util.Logging;
 
 namespace Semmle.Util
 {
@@ -275,14 +276,14 @@ namespace Semmle.Util
         /// <summary>
         /// Creates a build script that downloads the specified file.
         /// </summary>
-        public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback) =>
+        public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback, ILogger logger) =>
             Create(actions =>
             {
                 if (actions.GetDirectoryName(fileName) is string dir && !string.IsNullOrWhiteSpace(dir))
                     actions.CreateDirectory(dir);
                 try
                 {
-                    actions.DownloadFile(address, fileName);
+                    actions.DownloadFile(address, fileName, logger);
                     return 0;
                 }
                 catch (Exception e)

csharp/extractor/Semmle.Util/FileUtils.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using System.Net.Http;
 using System.Security.Cryptography;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using Semmle.Util.Logging;
 
@@ -86,32 +87,78 @@ namespace Semmle.Util
         }
 
         /// <summary>
-        /// Computes the hash of <paramref name="filePath"/>.
+        /// Computes the hash of the file at <paramref name="filePath"/>.
         /// </summary>
         public static string ComputeFileHash(string filePath)
         {
             using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
-            using var shaAlg = SHA256.Create();
-            var sha = shaAlg.ComputeHash(fileStream);
+            var sha = SHA256.HashData(fileStream);
+            return GetHashString(sha);
+        }
+
+        /// <summary>
+        /// Computes the hash of <paramref name="input"/>.
+        /// </summary>
+        public static string ComputeHash(string input)
+        {
+            var bytes = Encoding.Unicode.GetBytes(input);
+            var sha = MD5.HashData(bytes); // MD5 to keep it shorter than SHA256
+            return GetHashString(sha).ToUpper();
+        }
+
+        private static string GetHashString(byte[] sha)
+        {
             var hex = new StringBuilder(sha.Length * 2);
             foreach (var b in sha)
+            {
                 hex.AppendFormat("{0:x2}", b);
+            }
             return hex.ToString();
         }
 
-        private static async Task DownloadFileAsync(string address, string filename)
+        private static async Task DownloadFileAsync(string address, string filename, HttpClient httpClient, CancellationToken token)
         {
-            using var httpClient = new HttpClient();
-            using var contentStream = await httpClient.GetStreamAsync(address);
+            using var contentStream = await httpClient.GetStreamAsync(address, token);
             using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
-            await contentStream.CopyToAsync(stream);
+            await contentStream.CopyToAsync(stream, token);
+        }
+
+        private static void DownloadFileWithRetry(string address, string fileName, int tryCount, int timeoutMilliSeconds, ILogger logger)
+        {
+            logger.LogDebug($"Downloading {address} to {fileName}.");
+            using HttpClient client = new();
+
+            for (var i = 0; i < tryCount; i++)
+            {
+                logger.LogDebug($"Attempt {i + 1} of {tryCount}. Timeout: {timeoutMilliSeconds} ms.");
+                using var cts = new CancellationTokenSource();
+                cts.CancelAfter(timeoutMilliSeconds);
+                try
+                {
+                    DownloadFileAsync(address, fileName, client, cts.Token).GetAwaiter().GetResult();
+                    logger.LogDebug($"Downloaded {address} to {fileName}.");
+                    return;
+                }
+                catch (Exception exc)
+                {
+                    logger.LogDebug($"Failed to download {address} to {fileName}. Exception: {exc.Message}");
+                    timeoutMilliSeconds *= 2;
+
+                    if (i == tryCount - 1)
+                    {
+                        logger.LogDebug($"Failed to download {address} to {fileName} after {tryCount} attempts.");
+                        // Rethrowing the last exception
+                        throw;
+                    }
+                }
+            }
         }
 
         /// <summary>
         /// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
         /// </summary>
-        public static void DownloadFile(string address, string fileName) =>
-           DownloadFileAsync(address, fileName).GetAwaiter().GetResult();
+        public static void DownloadFile(string address, string fileName, ILogger logger) =>
+           DownloadFileWithRetry(address, fileName, tryCount: 3, timeoutMilliSeconds: 10000, logger);
 
         public static string ConvertPathToSafeRelativePath(string path)
         {

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.7.22
+version: 1.7.23-dev
 groups:
   - csharp
   - solorigate

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.7.22
+version: 1.7.23-dev
 groups:
   - csharp
   - solorigate

csharp/ql/integration-tests/all-platforms/autobuild/test.py

@@ -1,3 +1,2 @@
-from create_database_utils import *
-
-run_codeql_database_create([], lang="csharp")
+def test(codeql, csharp):
+    codeql.database.create()

csharp/ql/integration-tests/all-platforms/binlog/test.py

@@ -1,7 +1,6 @@
-import subprocess
-from create_database_utils import *
-from diagnostics_test_utils import *
+import commands
 
-subprocess.check_call(["dotnet", "build", "test.sln", "/bl:test.binlog"])
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none", "-Obinlog=test.binlog"])
-check_diagnostics()
+
+def test(codeql, csharp):
+    commands.run(["dotnet", "build", "test.sln", "/bl:test.binlog"])
+    codeql.database.create(build_mode="none", extractor_option="binlog=test.binlog")

csharp/ql/integration-tests/all-platforms/conditional_compilation/Methods.expected

@@ -0,0 +1,4 @@
+| Program.cs:3:24:3:27 | Main | 2 |
+| Program.cs:10:17:10:33 | GreetConditional1 | 2 |
+| Program.cs:19:17:19:21 | Greet | 2 |
+| Program.cs:25:17:25:33 | GreetConditional2 | 2 |

csharp/ql/integration-tests/all-platforms/conditional_compilation/Methods.ql

@@ -0,0 +1,5 @@
+import csharp
+
+from Method m
+where m.fromSource()
+select m, count(m.getBody())

csharp/ql/integration-tests/all-platforms/conditional_compilation/Program.cs

@@ -0,0 +1,40 @@
+public class Test
+{
+    public static void Main()
+    {
+        Greet();
+        GreetConditional1();
+        GreetConditional2();
+    }
+
+    static void GreetConditional1()
+    {
+#if A
+    Console.WriteLine("Hello, A!");
+#elif C
+    Console.WriteLine("Hello, C!");
+#endif
+    }
+
+    static void Greet()
+    {
+        Console.WriteLine("Hello, World!");
+    }
+
+#if A
+    static void GreetConditional2()
+    {
+        Console.WriteLine("Hello, A!");
+    }
+#elif B
+    static void GreetConditional2()
+    {
+        Console.WriteLine("Hello, B!");
+    }
+#else
+    static void GreetConditional2()
+    {
+        Console.WriteLine("Hello, Others!");
+    }
+#endif
+}

csharp/ql/integration-tests/all-platforms/conditional_compilation/global.json

@@ -0,0 +1,5 @@
+{
+    "sdk": {
+        "version": "8.0.101"
+    }
+}

csharp/ql/integration-tests/all-platforms/conditional_compilation/test.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Remove="**/*.cs" />
+    <Compile Include="Program.cs" />
+  </ItemGroup>
+
+</Project>

csharp/ql/integration-tests/all-platforms/conditional_compilation/test.py

@@ -0,0 +1,8 @@
+import os
+
+
+def test(codeql, csharp):
+    os.environ["CODEQL_EXTRACTOR_CSHARP_OPTION_TRAP_COMPRESSION"] = "none"
+    codeql.database.create(
+        command=["dotnet build /p:DefineConstants=A", "dotnet build /p:DefineConstants=B"]
+    )

csharp/ql/integration-tests/all-platforms/cshtml/test.py

@@ -1,3 +1,2 @@
-from create_database_utils import *
-
-run_codeql_database_create(['dotnet build'], lang="csharp")
+def test(codeql, csharp):
+    codeql.database.create(command="dotnet build")

csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.expected

@@ -1,4 +1,4 @@
 | Program.cs |
 | Views/Home/Index.cshtml |
-| _semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_Views_Home_Index_cshtml.g.cs |
 | test-db/working/implicitUsings/GlobalUsings.g.cs |
+| test-db/working/razor/EC52D77FE9BF67AD10C5C3F248392316/Microsoft.CodeAnalysis.Razor.Compiler/Microsoft.NET.Sdk.Razor.SourceGenerators.RazorSourceGenerator/[...]_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_test_test_Views_Home_Index_cshtml.g.cs |

csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.ql

@@ -2,16 +2,17 @@ import csharp
 
 private string getPath(File f) {
   result = f.getRelativePath() and
-  not exists(
-    result
-        .indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_")
-  )
+  not exists(result.indexOf("_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_"))
   or
-  exists(int index |
-    index =
+  exists(int index1, int index2, string pattern |
+    pattern = "Microsoft.NET.Sdk.Razor.SourceGenerators.RazorSourceGenerator" and
+    index1 = f.getRelativePath().indexOf(pattern) and
+    index2 =
       f.getRelativePath()
-          .indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_") and
-    result = f.getRelativePath().substring(index, f.getRelativePath().length())
+          .indexOf("_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_") and
+    result =
+      f.getRelativePath().substring(0, index1 + pattern.length()) + "/[...]" +
+        f.getRelativePath().substring(index2, f.getRelativePath().length())
   )
 }
 

csharp/ql/integration-tests/all-platforms/cshtml_standalone/test.py

@@ -1,4 +1,2 @@
-import os
-from create_database_utils import *
-
-run_codeql_database_create(lang="csharp", extra_args=["--build-mode=none"])
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/all-platforms/cshtml_standalone_disabled/Files.expected

@@ -1,2 +1,2 @@
-| Program.cs |
-| test-db/working/implicitUsings/GlobalUsings.g.cs |
+| Program.cs:0:0:0:0 | Program.cs |
+| test-db/working/implicitUsings/GlobalUsings.g.cs:0:0:0:0 | test-db/working/implicitUsings/GlobalUsings.g.cs |

csharp/ql/integration-tests/all-platforms/cshtml_standalone_disabled/Files.ql

@@ -1,20 +1,5 @@
 import csharp
 
-private string getPath(File f) {
-  result = f.getRelativePath() and
-  not exists(
-    result
-        .indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_")
-  )
-  or
-  exists(int index |
-    index =
-      f.getRelativePath()
-          .indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_") and
-    result = f.getRelativePath().substring(index, f.getRelativePath().length())
-  )
-}
-
 from File f
 where f.fromSource() or f.getExtension() = "cshtml"
-select getPath(f)
+select f

csharp/ql/integration-tests/all-platforms/cshtml_standalone_disabled/test.py

@@ -1,5 +1,6 @@
 import os
-from create_database_utils import *
 
-os.environ['CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_WEB_VIEWS'] = 'false'
-run_codeql_database_create(lang="csharp", extra_args=["--build-mode=none"])
+
+def test(codeql, csharp):
+    os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_WEB_VIEWS"] = "false"
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/all-platforms/cshtml_standalone_flowsteps/test.py

@@ -1,4 +1,2 @@
-import os
-from create_database_utils import *
-
-run_codeql_database_create(lang="csharp", extra_args=["--build-mode=none"])
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/all-platforms/cshtml_standalone_net6/Files.expected

@@ -1,4 +1,4 @@
 | Program.cs |
 | Views/Home/Index.cshtml |
-| _semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_net6_Views_Home_Index_cshtml.g.cs |
 | test-db/working/implicitUsings/GlobalUsings.g.cs |
+| test-db/working/razor/EC52D77FE9BF67AD10C5C3F248392316/[...]/Microsoft.NET.Sdk.Razor.SourceGenerators.RazorSourceGenerator/[...]_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_net6_test_test_Views_Home_Index_cshtml.g.cs |

csharp/ql/integration-tests/all-platforms/cshtml_standalone_net6/Files.ql

@@ -2,16 +2,21 @@ import csharp
 
 private string getPath(File f) {
   result = f.getRelativePath() and
-  not exists(
-    result
-        .indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_")
-  )
+  not exists(result.indexOf("_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_"))
   or
-  exists(int index |
-    index =
+  exists(int index0, int index1, int index2, string pattern0, string pattern1 |
+    // TODO: Remove index0 and pattern0. Currently there's some instability in the path depending on which dotnet SDK is being used. (See issue #448)
+    pattern0 = "EC52D77FE9BF67AD10C5C3F248392316" and
+    index0 = f.getRelativePath().indexOf(pattern0) and
+    pattern1 = "Microsoft.NET.Sdk.Razor.SourceGenerators.RazorSourceGenerator" and
+    index1 = f.getRelativePath().indexOf(pattern1) and
+    index2 =
       f.getRelativePath()
-          .indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_") and
-    result = f.getRelativePath().substring(index, f.getRelativePath().length())
+          .indexOf("_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_") and
+    result =
+      f.getRelativePath().substring(0, index0 + pattern0.length()) + "/[...]/" +
+        f.getRelativePath().substring(index1, index1 + pattern1.length()) + "/[...]" +
+        f.getRelativePath().substring(index2, f.getRelativePath().length())
   )
 }
 

csharp/ql/integration-tests/all-platforms/cshtml_standalone_net6/test.py

@@ -1,4 +1,2 @@
-import os
-from create_database_utils import *
-
-run_codeql_database_create(lang="csharp", extra_args=["--build-mode=none"])
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/all-platforms/diag_dotnet_incompatible/test.py

@@ -1,5 +1,2 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
-run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
-check_diagnostics()
+def test(codeql, csharp):
+    codeql.database.create(_assert_failure=True)

csharp/ql/integration-tests/all-platforms/diag_missing_project_files/test.py

@@ -1,5 +1,2 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
-run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
-check_diagnostics()
+def test(codeql, csharp):
+    codeql.database.create(_assert_failure=True)

csharp/ql/integration-tests/all-platforms/diag_missing_xamarin_sdk/test.py

@@ -1,5 +1,2 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
-run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
-check_diagnostics()
+def test(codeql, csharp):
+    codeql.database.create(_assert_failure=True)

csharp/ql/integration-tests/all-platforms/diag_recursive_generics/test.py

@@ -1,3 +1,2 @@
-from create_database_utils import *
-
-run_codeql_database_create(['dotnet build'], lang="csharp")
+def test(codeql, csharp):
+    codeql.database.create(command="dotnet build")

csharp/ql/integration-tests/all-platforms/dotnet_build/test.py

@@ -1,16 +1,21 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
 def check_build_out(msg, s):
-    if "[build-stdout] " + msg not in s:
-        raise Exception("The C# tracer did not interpret the dotnet path-to-application command correctly.")
+    assert (
+        "[build-stdout] " + msg in s
+    ), f"The C# tracer did not interpret the dotnet path-to-application command correctly."
+
+
+def test1(codeql, csharp):
+    codeql.database.create(command="dotnet build")
 
-run_codeql_database_create(['dotnet build'], test_db="test1-db", lang="csharp")
-check_diagnostics(test_db="test1-db")
 
 # This test checks that we don't inject any flags when running the application using `dotnet`
-my_dir = "my_program"
-my_abs_path = os.path.abspath(f"{my_dir}/dotnet_build.dll")
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test1-db', 'dotnet build -o my_program', f'dotnet {my_abs_path} build is not a subcommand'], "test2-db", "csharp")
-check_build_out("<arguments>build,is,not,a,subcommand</arguments>", s)
-check_diagnostics(test_db="test2-db")
+def test2(codeql, csharp, cwd):
+    s = codeql.database.create(
+        command=[
+            "dotnet build -o my_program",
+            f"dotnet {cwd / 'my_program'}/dotnet_build.dll build is not a subcommand",
+        ],
+        _capture="stdout",
+    )
+
+    check_build_out("<arguments>build,is,not,a,subcommand</arguments>", s)

csharp/ql/integration-tests/all-platforms/dotnet_no_args_inject/test.py

@@ -1,9 +1,9 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
-# the tracer configuration should not inject the extra command-line arguments for these commands
-# and they should therefore run successfully
-run_codeql_database_init(lang="csharp")
-# this command fails on Windows for some reason, so we comment it out for now
-# run_codeql_database_trace_command(['dotnet', 'tool', 'search', 'publish'])
-run_codeql_database_trace_command(['dotnet', 'new', 'console', '--force', '--name', 'build', '--output', '.'])
+def test(codeql, csharp):
+    codeql.database.init("test-db", source_root=".")
+    # the tracer configuration should not inject the extra command-line arguments for these commands
+    # and they should therefore run successfully
+    # this command fails on Windows for some reason, so we comment it out for now
+    # run_codeql_database_trace_command(['dotnet', 'tool', 'search', 'publish'])
+    codeql.database.trace_command(
+        "test-db", "dotnet", "new", "console", "--force", "--name", "build", "--output", "."
+    )

csharp/ql/integration-tests/all-platforms/dotnet_pack/test.py

@@ -1,11 +1,8 @@
 import os
-from create_database_utils import *
-from diagnostics_test_utils import *
 
-run_codeql_database_create(['dotnet pack -o nugetpackage'], db=None, lang="csharp")
 
-## Check that the NuGet package is created.
-if not os.path.isfile("nugetpackage/dotnet_pack.1.0.0.nupkg"):
-    raise Exception("The NuGet package was not created.")
-
-check_diagnostics()
+def test(codeql, csharp):
+    codeql.database.create(command="dotnet pack -o nugetpackage")
+    assert os.path.isfile(
+        "nugetpackage/dotnet_pack.1.0.0.nupkg"
+    ), "The NuGet package was not created."

csharp/ql/integration-tests/all-platforms/dotnet_publish/test.py

@@ -1,12 +1,7 @@
 import os
-from create_database_utils import *
-from diagnostics_test_utils import *
 
-artifacts = 'bin/Temp'
-run_codeql_database_create([f"dotnet publish -o {artifacts}"], db=None, lang="csharp")
 
-## Check that the publish folder is created.
-if not os.path.isdir(artifacts):
-    raise Exception("The publish artifact folder was not created.")
-
-check_diagnostics()
+def test(codeql, csharp):
+    artifacts = "bin/Temp"
+    codeql.database.create(command=f"dotnet publish -o {artifacts}")
+    assert os.path.isdir(artifacts), "The publish artifact folder was not created."

csharp/ql/integration-tests/all-platforms/dotnet_run/test.py

@@ -1,57 +1,69 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
 def check_build_out(msg, s):
-    if "[build-stdout] " + msg not in s:
-        raise Exception("The C# tracer did not interpret the 'dotnet run' command correctly")
+    assert (
+        "[build-stdout] " + msg in s
+    ), "The C# tracer did not interpret the 'dotnet run' command correctly"
+
 
 # no arguments
-s = run_codeql_database_create_stdout(['dotnet run'], "test-db", "csharp")
-check_build_out("Default reply", s)
-check_diagnostics()
+def test_no_args(codeql, csharp):
+    s = codeql.database.create(command="dotnet run", _capture="stdout")
+    check_build_out("Default reply", s)
+
 
 # no arguments, but `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test-db', 'dotnet run --'], "test2-db", "csharp")
-check_build_out("Default reply", s)
-check_diagnostics(test_db="test2-db")
+def test_no_arg_dash_dash(codeql, csharp):
+    s = codeql.database.create(command="dotnet run --", _capture="stdout")
+    check_build_out("Default reply", s)
+
 
 # one argument, no `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test2-db', 'dotnet run hello'], "test3-db", "csharp")
-check_build_out("Default reply", s)
-check_diagnostics(test_db="test3-db")
+def test_one_arg_no_dash_dash(codeql, csharp):
+    s = codeql.database.create(command="dotnet run hello", _capture="stdout")
+    check_build_out("Default reply", s)
+
 
 # one argument, but `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test3-db', 'dotnet run -- hello'], "test4-db", "csharp")
-check_build_out("Default reply", s)
-check_diagnostics(test_db="test4-db")
+def test_one_arg_dash_dash(codeql, csharp):
+    s = codeql.database.create(command="dotnet run -- hello", _capture="stdout")
+    check_build_out("Default reply", s)
+
 
 # two arguments, no `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test4-db', 'dotnet run hello world'], "test5-db", "csharp")
-check_build_out("hello, world", s)
-check_diagnostics(test_db="test5-db")
+def test_two_args_no_dash_dash(codeql, csharp):
+    s = codeql.database.create(command="dotnet run hello world", _capture="stdout")
+    check_build_out("hello, world", s)
+
 
 # two arguments, and `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test5-db', 'dotnet run -- hello world'], "test6-db", "csharp")
-check_build_out("hello, world", s)
-check_diagnostics(test_db="test6-db")
+def test_two_args_dash_dash(codeql, csharp):
+    s = codeql.database.create(command="dotnet run -- hello world", _capture="stdout")
+    check_build_out("hello, world", s)
+
 
 # shared compilation enabled; tracer should override by changing the command
 # to `dotnet run -p:UseSharedCompilation=true -p:UseSharedCompilation=false -- hello world`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test6-db', 'dotnet run -p:UseSharedCompilation=true -- hello world'], "test7-db", "csharp")
-check_build_out("hello, world", s)
-check_diagnostics(test_db="test7-db")
+def test_shared_compilation(codeql, csharp):
+    s = codeql.database.create(
+        command="dotnet run -p:UseSharedCompilation=true -- hello world", _capture="stdout"
+    )
+    check_build_out("hello, world", s)
+
 
 # option passed into `dotnet run`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test7-db', 'dotnet build', 'dotnet run --no-build hello world'], "test8-db", "csharp")
-check_build_out("hello, world", s)
-check_diagnostics(test_db="test8-db")
+def test_option(codeql, csharp):
+    s = codeql.database.create(
+        command=["dotnet build", "dotnet run --no-build hello world"], _capture="stdout"
+    )
+    check_build_out("hello, world", s)
+
 
 # two arguments, no '--' (first argument quoted)
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test8-db', 'dotnet run "hello world part1" part2'], "test9-db", "csharp")
-check_build_out("hello world part1, part2", s)
-check_diagnostics(test_db="test9-db")
+def test_two_args_no_dash_dash_quote_first(codeql, csharp):
+    s = codeql.database.create(command='dotnet run "hello world" part2', _capture="stdout")
+    check_build_out("hello world, part2", s)
+
 
 # two arguments, no '--' (second argument quoted) and using dotnet to execute dotnet
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test9-db', 'dotnet dotnet run part1 "hello world part2"'], "test10-db", "csharp")
-check_build_out("part1, hello world part2", s)
-check_diagnostics(test_db="test10-db")
+def test_two_args_no_dash_dash_quote_second(codeql, csharp):
+    s = codeql.database.create(command='dotnet dotnet run hello "world part2"', _capture="stdout")
+    check_build_out("hello, world part2", s)

csharp/ql/integration-tests/all-platforms/msbuild/test.py

@@ -1,6 +1,3 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
-# force CodeQL to use MSBuild by setting `LGTM_INDEX_MSBUILD_TARGET`
-run_codeql_database_create([], db=None, lang="csharp", extra_env={ 'LGTM_INDEX_MSBUILD_TARGET': 'Build' })
-check_diagnostics()
+def test(codeql, csharp):
+    # force CodeQL to use MSBuild by setting `LGTM_INDEX_MSBUILD_TARGET`
+    codeql.database.create(_env={"LGTM_INDEX_MSBUILD_TARGET": "Build"})

csharp/ql/integration-tests/all-platforms/source_generator/test.py

@@ -1,3 +1,2 @@
-from create_database_utils import *
-
-run_codeql_database_create(['dotnet build'], lang="csharp")
+def test(codeql, csharp):
+    codeql.database.create(command="dotnet build")

csharp/ql/integration-tests/all-platforms/standalone/DatabaseQualityDiagnostics.expected

@@ -0,0 +1,6 @@
+diagnosticAttributes
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityCliSummaryTable | true |
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityStatusPage | true |
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityTelemetry | true |
+#select
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | 1 |

csharp/ql/integration-tests/all-platforms/standalone/DatabaseQualityDiagnostics.qlref

@@ -0,0 +1 @@
+Telemetry/DatabaseQualityDiagnostics.ql

csharp/ql/integration-tests/all-platforms/standalone/Diag.expected

@@ -1,7 +1,11 @@
 extractorMessages
-| 5 |
+| 6 |
 compilerDiagnostics
 | 4 |
+extractorMessagesLeachedLimit
+| Program.cs:1:1:1:0 | Stopped logging extractor messages after reaching 5 |
 compilationInfo
 | Compiler diagnostic count for CS0103 | 3.0 |
 | Compiler diagnostic count for CS8019 | 7.0 |
+| Extractor message count for group 'Error' | 8.0 |
+| Extractor message count for group 'Warning' | 1.0 |

csharp/ql/integration-tests/all-platforms/standalone/Diag.ql

@@ -5,9 +5,14 @@ query predicate extractorMessages(int c) { c = count(ExtractorMessage msg) }
 
 query predicate compilerDiagnostics(int c) { c = count(Diagnostic diag) }
 
+query predicate extractorMessagesLeachedLimit(ExtractorMessage msg) {
+  msg.getText().indexOf("Stopped logging") = 0
+}
+
 query predicate compilationInfo(string key, float value) {
   exists(Compilation c, string infoValue |
-    infoValue = c.getInfo(key) and key.matches("Compiler diagnostic count for%")
+    infoValue = c.getInfo(key) and
+    key.matches(["Compiler diagnostic count for%", "Extractor message count for group%"])
   |
     value = infoValue.toFloat()
   )

csharp/ql/integration-tests/all-platforms/standalone/test.py

@@ -1,9 +1,7 @@
 import os
-from create_database_utils import *
-from diagnostics_test_utils import *
 
-os.environ['CODEQL_EXTRACTOR_CSHARP_OPTION_COMPILER_DIAGNOSTIC_LIMIT'] = '2'
-os.environ['CODEQL_EXTRACTOR_CSHARP_OPTION_MESSAGE_LIMIT'] = '5'
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
 
-check_diagnostics()
+def test(codeql, csharp):
+    os.environ["CODEQL_EXTRACTOR_CSHARP_OPTION_COMPILER_DIAGNOSTIC_LIMIT"] = "2"
+    os.environ["CODEQL_EXTRACTOR_CSHARP_OPTION_MESSAGE_LIMIT"] = "5"
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/all-platforms/standalone_buildless_option/DatabaseQualityDiagnostics.expected

@@ -0,0 +1,2 @@
+diagnosticAttributes
+#select