Upgrade to bazel 7.3.1.

Also flip the flag for the repo separator.

.bazelrc

@@ -1,5 +1,17 @@
+# allow automatic platform-specific configurations (:linux, :macos, :windows)
 common --enable_platform_specific_config
+
+# do not shut down the bazel server on idleness
+startup --max_idle_secs=0
 common --enable_bzlmod
+# use stricter environment filtering for actions
+build --incompatible_strict_action_env
+# Enable verbose failures so debugging the build is easier
+build --verbose_failures
+# print test output, like sembuild does.
+# Set to `errors` if this is too verbose.
+test --test_output all
+
 # because we use --override_module with `%workspace%`, the lock file is not stable
 common --lockfile_mode=off
 
@@ -24,5 +36,10 @@ common --registry=file:///%workspace%/misc/bazel/registry
 common --registry=https://bcr.bazel.build
 
 common --@rules_dotnet//dotnet/settings:strict_deps=false
+common --experimental_isolated_extension_usages
+# performance improvement on Windows
+common --incompatible_use_plus_in_repo_names
+
+
 
 try-import %workspace%/local.bazelrc

.bazelrc.internal

@@ -8,3 +8,4 @@ common --registry=https://bcr.bazel.build
 # its implementation packages without providing any code itself.
 # We either can depend on internal implementation details, or turn of strict deps.
 common --@rules_dotnet//dotnet/settings:strict_deps=false
+common --experimental_isolated_extension_usages

.bazelversion

@@ -1 +1 @@
-7.2.1
+7.3.1

MODULE.bazel

@@ -1,6 +1,7 @@
 module(
-    name = "codeql",
+    name = "ql",
     version = "0.0",
+    repo_name = "codeql",
 )
 
 # this points to our internal repository when `codeql` is checked out as a submodule thereof
@@ -30,11 +31,13 @@ bazel_dep(name = "rules_rust", version = "0.49.1")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
-crate = use_extension(
+# crate_py but shortened due to Windows file path considerations
+cp = use_extension(
     "@rules_rust//crate_universe:extension.bzl",
     "crate",
+    isolate = True,
 )
-crate.from_cargo(
+cp.from_cargo(
     name = "py_deps",
     cargo_lockfile = "//python/extractor/tsg-python:Cargo.lock",
     manifests = [
@@ -42,15 +45,23 @@ crate.from_cargo(
         "//python/extractor/tsg-python/tsp:Cargo.toml",
     ],
 )
-crate.from_cargo(
-    name = "ruby_deps",
+use_repo(cp, "py_deps")
+
+# crate_ruby, but shortened due to windows file paths
+r = use_extension(
+    "@rules_rust//crate_universe:extension.bzl",
+    "crate",
+    isolate = True,
+)
+r.from_cargo(
+    name = "rd",
     cargo_lockfile = "//ruby/extractor:Cargo.lock",
     manifests = [
         "//ruby/extractor:Cargo.toml",
         "//ruby/extractor/codeql-extractor-fake-crate:Cargo.toml",
     ],
 )
-use_repo(crate, "py_deps", "ruby_deps")
+use_repo(r, ruby_deps = "rd")
 
 dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet")
 dotnet.toolchain(dotnet_version = "8.0.101")

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 1.4.0
+version: 1.4.1-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/dataflow/ExternalFlow.qll

@@ -953,21 +953,3 @@ private class SummarizedCallableAdapter extends SummarizedCallable {
     interpretSummary(this, _, _, _, provenance, _)
   }
 }
-
-// adapter class for converting Mad neutrals to `NeutralCallable`s
-private class NeutralCallableAdapter extends NeutralCallable {
-  string kind;
-  string provenance_;
-
-  NeutralCallableAdapter() {
-    // Neutral models have not been implemented for CPP.
-    none() and
-    exists(this) and
-    exists(kind) and
-    exists(provenance_)
-  }
-
-  override string getKind() { result = kind }
-
-  override predicate hasProvenance(Provenance provenance) { provenance = provenance_ }
-}

cpp/ql/src/Security/CWE/CWE-190/TaintedAllocationSize.c

@@ -1,11 +1,9 @@
 int factor = atoi(getenv("BRANCHING_FACTOR"));
 
-// GOOD: Prevent overflow by checking the input
-if (factor < 0 || factor > 1000) {
-    log("Factor out of range (%d)\n", factor);
-    return -1;
-}
-
-// This line can allocate too little memory if factor
-// is very large.
+// BAD: This can allocate too little memory if factor is very large due to overflow.
 char **root_node = (char **) malloc(factor * sizeof(char *));
+
+// GOOD: Prevent overflow and unbounded allocation size by checking the input.
+if (factor > 0 && factor <= 1000) {
+    char **root_node = (char **) malloc(factor * sizeof(char *));
+}

cpp/ql/src/Security/CWE/CWE-190/TaintedAllocationSize.qhelp

@@ -3,12 +3,16 @@
   "qhelp.dtd">
 <qhelp>
 <overview>
-<p>This code calculates an allocation size by multiplying a user input
-by a <code>sizeof</code> expression. Since the user input has no
-apparent guard on its magnitude, this multiplication can
-overflow. When an integer multiply overflows in C, the result can wrap
-around and be much smaller than intended. A later attempt to put data
-into the allocated buffer can then overflow.</p>
+
+<p>This code allocates memory using a size value based on user input,
+with no apparent bound on its magnitude being established. This allows
+for arbitrary amounts of memory to be allocated.</p>
+
+<p>If the allocation size is calculated by multiplying user input by a
+<code>sizeof</code> expression, the multiplication can overflow. When
+an integer multiplication overflows in C, the result wraps around and
+can be much smaller than intended. A later attempt to write data into
+the allocated memory can then be out of bounds.</p>
 
 </overview>
 <recommendation>

cpp/ql/src/Security/CWE/CWE-190/TaintedAllocationSize.ql

@@ -1,7 +1,7 @@
 /**
- * @name Overflow in uncontrolled allocation size
- * @description Allocating memory with a size controlled by an external
- *              user can result in integer overflow.
+ * @name Uncontrolled allocation size
+ * @description Allocating memory with a size controlled by an external user can result in
+ *              arbitrary amounts of memory being allocated.
  * @kind path-problem
  * @problem.severity error
  * @security-severity 8.1
@@ -104,5 +104,6 @@ where
   isFlowSource(source.getNode(), taintCause) and
   TaintedAllocationSize::flowPath(source, sink) and
   allocSink(alloc, sink.getNode())
-select alloc, source, sink, "This allocation size is derived from $@ and might overflow.",
+select alloc, source, sink,
+  "This allocation size is derived from $@ and could allocate arbitrary amounts of memory.",
   source.getNode(), "user input (" + taintCause + ")"

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 1.2.0
+version: 1.2.1-dev
 groups:
   - cpp
   - queries

cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/TaintedAllocationSize/TaintedAllocationSize.expected

@@ -79,21 +79,21 @@ nodes
 | test.cpp:356:35:356:38 | size | semmle.label | size |
 subpaths
 #select
-| test.cpp:43:31:43:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:43:38:43:44 | tainted | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:44:31:44:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:44:38:44:63 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:46:31:46:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:46:38:46:63 | ... + ... | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:49:25:49:30 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:49:32:49:35 | size | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:50:17:50:30 | new[] | test.cpp:39:27:39:30 | **argv | test.cpp:50:17:50:30 | size | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:53:21:53:27 | call to realloc | test.cpp:39:27:39:30 | **argv | test.cpp:53:35:53:60 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
-| test.cpp:128:17:128:22 | call to malloc | test.cpp:124:18:124:31 | *call to getenv | test.cpp:128:24:128:41 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:124:18:124:31 | *call to getenv | user input (an environment variable) |
-| test.cpp:135:3:135:8 | call to malloc | test.cpp:133:19:133:32 | *call to getenv | test.cpp:135:10:135:27 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:133:19:133:32 | *call to getenv | user input (an environment variable) |
-| test.cpp:152:4:152:9 | call to malloc | test.cpp:148:20:148:33 | *call to getenv | test.cpp:152:11:152:28 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:148:20:148:33 | *call to getenv | user input (an environment variable) |
-| test.cpp:231:14:231:19 | call to malloc | test.cpp:237:24:237:37 | *call to getenv | test.cpp:231:21:231:21 | s | This allocation size is derived from $@ and might overflow. | test.cpp:237:24:237:37 | *call to getenv | user input (an environment variable) |
-| test.cpp:239:2:239:7 | call to malloc | test.cpp:237:24:237:37 | *call to getenv | test.cpp:239:9:239:18 | local_size | This allocation size is derived from $@ and might overflow. | test.cpp:237:24:237:37 | *call to getenv | user input (an environment variable) |
-| test.cpp:241:2:241:7 | call to malloc | test.cpp:211:14:211:27 | *call to getenv | test.cpp:241:9:241:24 | call to get_tainted_size | This allocation size is derived from $@ and might overflow. | test.cpp:211:14:211:27 | *call to getenv | user input (an environment variable) |
-| test.cpp:245:2:245:9 | call to my_alloc | test.cpp:237:24:237:37 | *call to getenv | test.cpp:245:11:245:20 | local_size | This allocation size is derived from $@ and might overflow. | test.cpp:237:24:237:37 | *call to getenv | user input (an environment variable) |
-| test.cpp:263:4:263:9 | call to malloc | test.cpp:259:20:259:33 | *call to getenv | test.cpp:263:11:263:29 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:259:20:259:33 | *call to getenv | user input (an environment variable) |
-| test.cpp:291:4:291:9 | call to malloc | test.cpp:251:18:251:31 | *call to getenv | test.cpp:291:11:291:28 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:251:18:251:31 | *call to getenv | user input (an environment variable) |
-| test.cpp:308:3:308:8 | call to malloc | test.cpp:251:18:251:31 | *call to getenv | test.cpp:308:10:308:27 | ... * ... | This allocation size is derived from $@ and might overflow. | test.cpp:251:18:251:31 | *call to getenv | user input (an environment variable) |
-| test.cpp:355:25:355:33 | call to MyMalloc1 | test.cpp:353:18:353:31 | *call to getenv | test.cpp:355:35:355:38 | size | This allocation size is derived from $@ and might overflow. | test.cpp:353:18:353:31 | *call to getenv | user input (an environment variable) |
-| test.cpp:356:25:356:33 | call to MyMalloc2 | test.cpp:353:18:353:31 | *call to getenv | test.cpp:356:35:356:38 | size | This allocation size is derived from $@ and might overflow. | test.cpp:353:18:353:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:43:31:43:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:43:38:43:44 | tainted | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:44:31:44:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:44:38:44:63 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:46:31:46:36 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:46:38:46:63 | ... + ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:49:25:49:30 | call to malloc | test.cpp:39:27:39:30 | **argv | test.cpp:49:32:49:35 | size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:50:17:50:30 | new[] | test.cpp:39:27:39:30 | **argv | test.cpp:50:17:50:30 | size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:53:21:53:27 | call to realloc | test.cpp:39:27:39:30 | **argv | test.cpp:53:35:53:60 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:39:27:39:30 | **argv | user input (a command-line argument) |
+| test.cpp:128:17:128:22 | call to malloc | test.cpp:124:18:124:31 | *call to getenv | test.cpp:128:24:128:41 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:124:18:124:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:135:3:135:8 | call to malloc | test.cpp:133:19:133:32 | *call to getenv | test.cpp:135:10:135:27 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:133:19:133:32 | *call to getenv | user input (an environment variable) |
+| test.cpp:152:4:152:9 | call to malloc | test.cpp:148:20:148:33 | *call to getenv | test.cpp:152:11:152:28 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:148:20:148:33 | *call to getenv | user input (an environment variable) |
+| test.cpp:231:14:231:19 | call to malloc | test.cpp:237:24:237:37 | *call to getenv | test.cpp:231:21:231:21 | s | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:237:24:237:37 | *call to getenv | user input (an environment variable) |
+| test.cpp:239:2:239:7 | call to malloc | test.cpp:237:24:237:37 | *call to getenv | test.cpp:239:9:239:18 | local_size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:237:24:237:37 | *call to getenv | user input (an environment variable) |
+| test.cpp:241:2:241:7 | call to malloc | test.cpp:211:14:211:27 | *call to getenv | test.cpp:241:9:241:24 | call to get_tainted_size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:211:14:211:27 | *call to getenv | user input (an environment variable) |
+| test.cpp:245:2:245:9 | call to my_alloc | test.cpp:237:24:237:37 | *call to getenv | test.cpp:245:11:245:20 | local_size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:237:24:237:37 | *call to getenv | user input (an environment variable) |
+| test.cpp:263:4:263:9 | call to malloc | test.cpp:259:20:259:33 | *call to getenv | test.cpp:263:11:263:29 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:259:20:259:33 | *call to getenv | user input (an environment variable) |
+| test.cpp:291:4:291:9 | call to malloc | test.cpp:251:18:251:31 | *call to getenv | test.cpp:291:11:291:28 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:251:18:251:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:308:3:308:8 | call to malloc | test.cpp:251:18:251:31 | *call to getenv | test.cpp:308:10:308:27 | ... * ... | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:251:18:251:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:355:25:355:33 | call to MyMalloc1 | test.cpp:353:18:353:31 | *call to getenv | test.cpp:355:35:355:38 | size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:353:18:353:31 | *call to getenv | user input (an environment variable) |
+| test.cpp:356:25:356:33 | call to MyMalloc2 | test.cpp:353:18:353:31 | *call to getenv | test.cpp:356:35:356:38 | size | This allocation size is derived from $@ and could allocate arbitrary amounts of memory. | test.cpp:353:18:353:31 | *call to getenv | user input (an environment variable) |

cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/TaintedAllocationSize/test.cpp

@@ -40,10 +40,10 @@ int main(int argc, char **argv) {
 	int tainted = atoi(argv[1]);
 
 	MyStruct *arr1 = (MyStruct *)malloc(sizeof(MyStruct)); // GOOD
-	MyStruct *arr2 = (MyStruct *)malloc(tainted); // DUBIOUS (not multiplied by anything)
+	MyStruct *arr2 = (MyStruct *)malloc(tainted); // BAD
 	MyStruct *arr3 = (MyStruct *)malloc(tainted * sizeof(MyStruct)); // BAD
 	MyStruct *arr4 = (MyStruct *)malloc(getTainted() * sizeof(MyStruct)); // BAD [NOT DETECTED]
-	MyStruct *arr5 = (MyStruct *)malloc(sizeof(MyStruct) + tainted); // DUBIOUS (not multiplied by anything)
+	MyStruct *arr5 = (MyStruct *)malloc(sizeof(MyStruct) + tainted); // BAD
 
 	int size = tainted * 8;
 	char *chars1 = (char *)malloc(size); // BAD

csharp/documentation/library-coverage/coverage.csv

@@ -10,7 +10,7 @@ Internal.IL,,,46,,,,,,,,,,,,,,,,,,,44,2
 Internal.Pgo,,,9,,,,,,,,,,,,,,,,,,,8,1
 Internal.TypeSystem,,,315,,,,,,,,,,,,,,,,,,,299,16
 JsonToItemsTaskFactory,,,10,,,,,,,,,,,,,,,,,,,10,
-Microsoft.Android.Build,,,16,,,,,,,,,,,,,,,,,,,16,
+Microsoft.Android.Build,,1,16,,,,,,,,,,,,,1,,,,,,16,
 Microsoft.Apple.Build,,,8,,,,,,,,,,,,,,,,,,,8,
 Microsoft.ApplicationBlocks.Data,28,,,,,,,,,,,,28,,,,,,,,,,
 Microsoft.CSharp,,,13,,,,,,,,,,,,,,,,,,,13,
@@ -19,9 +19,9 @@ Microsoft.DotNet.Build.Tasks,,,6,,,,,,,,,,,,,,,,,,,6,
 Microsoft.EntityFrameworkCore,6,,12,,,,,,,,,,6,,,,,,,,,,12
 Microsoft.Extensions.Caching.Distributed,,,10,,,,,,,,,,,,,,,,,,,10,
 Microsoft.Extensions.Caching.Memory,,,39,,,,,,,,,,,,,,,,,,,38,1
-Microsoft.Extensions.Configuration,,2,90,,,,,,,,,,,,,2,,,,,,89,1
+Microsoft.Extensions.Configuration,,3,90,,,,,,,,,,,,,3,,,,,,89,1
 Microsoft.Extensions.DependencyInjection,,,134,,,,,,,,,,,,,,,,,,,133,1
-Microsoft.Extensions.DependencyModel,,,18,,,,,,,,,,,,,,,,,,,18,
+Microsoft.Extensions.DependencyModel,,1,18,,,,,,,,,,,,,1,,,,,,18,
 Microsoft.Extensions.Diagnostics.Metrics,,,15,,,,,,,,,,,,,,,,,,,15,
 Microsoft.Extensions.FileProviders,,,15,,,,,,,,,,,,,,,,,,,15,
 Microsoft.Extensions.FileSystemGlobbing,,,18,,,,,,,,,,,,,,,,,,,16,2
@@ -41,5 +41,5 @@ MySql.Data.MySqlClient,48,,,,,,,,,,,,48,,,,,,,,,,
 Newtonsoft.Json,,,91,,,,,,,,,,,,,,,,,,,73,18
 ServiceStack,194,,7,27,,,,,75,,,,92,,,,,,,,,7,
 SourceGenerators,,,5,,,,,,,,,,,,,,,,,,,5,
-System,60,44,10614,,7,6,5,,,4,5,,33,2,,3,15,17,3,4,,8709,1905
+System,54,47,10626,,6,5,5,,,4,1,,33,2,,6,15,17,3,4,,8721,1905
 Windows.Security.Cryptography.Core,1,,,,,,,1,,,,,,,,,,,,,,,

csharp/documentation/library-coverage/coverage.rst

@@ -8,7 +8,7 @@ C# framework & library support
 
    Framework / library,Package,Flow sources,Taint & value steps,Sinks (total),`CWE-079` :sub:`Cross-site scripting`
    `ServiceStack <https://servicestack.net/>`_,"``ServiceStack.*``, ``ServiceStack``",,7,194,
-   System,"``System.*``, ``System``",44,10614,60,9
-   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.DotNet.Build.Tasks``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",54,1821,148,
-   Totals,,98,12442,402,9
+   System,"``System.*``, ``System``",47,10626,54,5
+   Others,"``Amazon.Lambda.APIGatewayEvents``, ``Amazon.Lambda.Core``, ``Dapper``, ``ILCompiler``, ``ILLink.RoslynAnalyzer``, ``ILLink.Shared``, ``ILLink.Tasks``, ``Internal.IL``, ``Internal.Pgo``, ``Internal.TypeSystem``, ``JsonToItemsTaskFactory``, ``Microsoft.Android.Build``, ``Microsoft.Apple.Build``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.Diagnostics.Tools.Pgo``, ``Microsoft.DotNet.Build.Tasks``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.Diagnostics.Metrics``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NET.WebAssembly.Webcil``, ``Microsoft.VisualBasic``, ``Microsoft.WebAssembly.Build.Tasks``, ``Microsoft.Win32``, ``Mono.Linker``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``, ``SourceGenerators``, ``Windows.Security.Cryptography.Core``",57,1821,148,
+   Totals,,104,12454,396,5
 

csharp/downgrades/15b989afd2bfc4743536fdb0958c1d8177a32600/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add unique constraint on preprocessor directive and compilation pairs
+compatibility: backwards

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs

@@ -189,13 +189,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// (together with used package information) required for compilation.
         /// </summary>
         /// <returns>True if parsing succeeds, otherwise false.</returns>
-        public bool TryParse(string json)
+        public bool TryParse(string json, string jsonPath)
         {
             try
             {
                 var obj = JObject.Parse(json);
-                AddPackageDependencies(obj, json);
-                AddFrameworkDependencies(obj, json);
+                AddPackageDependencies(obj, jsonPath);
+                AddFrameworkDependencies(obj, jsonPath);
                 return true;
             }
             catch (Exception e)
@@ -228,7 +228,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         {
             if (TryReadAllText(asset, logger, out var json))
             {
-                TryParse(json);
+                TryParse(json, asset);
             }
         }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -667,7 +667,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                     "Found unreachable Nuget feed in C# analysis with build-mode 'none'",
                     visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
                     markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
-                    severity: DiagnosticMessage.TspSeverity.Warning
+                    severity: DiagnosticMessage.TspSeverity.Note
                 ));
             }
             compilationInfoContainer.CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs

@@ -21,7 +21,6 @@ namespace Semmle.Extraction.CSharp.Entities
 
         protected override void Populate(TextWriter trapFile)
         {
-            // The below doesn't limit the extractor messages to the exact limit, but it's good enough.
             var key = diagnostic.Id;
             var messageCount = compilation.messageCounts.AddOrUpdate(key, 1, (_, c) => c + 1);
             if (messageCount > limit)

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/ElifDirective.cs

@@ -20,6 +20,7 @@ namespace Semmle.Extraction.CSharp.Entities
         public override void WriteId(EscapingTextWriter trapFile)
         {
             trapFile.WriteSubId(Context.CreateLocation(ReportingLocation));
+            trapFile.WriteSubId(start);
             trapFile.Write(Symbol.IsActive);
             trapFile.Write(',');
             trapFile.Write(Symbol.BranchTaken);

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/ElseDirective.cs

@@ -18,6 +18,7 @@ namespace Semmle.Extraction.CSharp.Entities
         public override void WriteId(EscapingTextWriter trapFile)
         {
             trapFile.WriteSubId(Context.CreateLocation(ReportingLocation));
+            trapFile.WriteSubId(start);
             trapFile.Write(Symbol.IsActive);
             trapFile.Write(',');
             trapFile.Write(Symbol.BranchTaken);

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/EndIfDirective.cs

@@ -13,6 +13,14 @@ namespace Semmle.Extraction.CSharp.Entities
             this.start = start;
         }
 
+        public override void WriteId(EscapingTextWriter trapFile)
+        {
+            trapFile.WriteSubId(Context.CreateLocation(ReportingLocation));
+            trapFile.WriteSubId(start);
+            trapFile.Write(Symbol.IsActive);
+            trapFile.Write(";trivia");
+        }
+
         protected override void PopulatePreprocessor(TextWriter trapFile)
         {
             trapFile.directive_endifs(this, start);

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -190,37 +190,29 @@ namespace Semmle.Extraction.CSharp
                 var transformedSourcePath = PathTransformer.Transform(sourcePath);
 
                 var trapPath = transformedSourcePath.GetTrapPath(Logger, options.TrapCompression);
-                var upToDate = false;
-
-                // compilation.Clone() is used to allow symbols to be garbage collected.
                 using var trapWriter = transformedSourcePath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
 
-                upToDate = FileIsUpToDate(sourcePath, trapWriter.TrapFile);
-
                 var currentTaskId = IncrementTaskCount();
                 ReportProgressTaskStarted(currentTaskId, sourcePath);
 
-                if (!upToDate)
+                var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
+                // Ensure that the file itself is populated in case the source file is totally empty
+                var root = tree.GetRoot();
+                Entities.File.Create(cx, root.SyntaxTree.FilePath);
+
+                var csNode = (CSharpSyntaxNode)root;
+                var directiveVisitor = new DirectiveVisitor(cx);
+                csNode.Accept(directiveVisitor);
+                foreach (var branch in directiveVisitor.BranchesTaken)
                 {
-                    var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
-                    // Ensure that the file itself is populated in case the source file is totally empty
-                    var root = tree.GetRoot();
-                    Entities.File.Create(cx, root.SyntaxTree.FilePath);
-
-                    var csNode = (CSharpSyntaxNode)root;
-                    var directiveVisitor = new DirectiveVisitor(cx);
-                    csNode.Accept(directiveVisitor);
-                    foreach (var branch in directiveVisitor.BranchesTaken)
-                    {
-                        cx.TrapStackSuffix.Add(branch);
-                    }
-                    csNode.Accept(new CompilationUnitVisitor(cx));
-                    cx.PopulateAll();
-                    CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
-                    cx.PopulateAll();
+                    cx.TrapStackSuffix.Add(branch);
                 }
+                csNode.Accept(new CompilationUnitVisitor(cx));
+                cx.PopulateAll();
+                CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
+                cx.PopulateAll();
 
-                ReportProgressTaskDone(currentTaskId, sourcePath, trapPath, stopwatch.Elapsed, upToDate ? AnalysisAction.UpToDate : AnalysisAction.Extracted);
+                ReportProgressTaskDone(currentTaskId, sourcePath, trapPath, stopwatch.Elapsed, AnalysisAction.Extracted);
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
@@ -268,12 +260,6 @@ namespace Semmle.Extraction.CSharp
             extractionTasks.Add(() => DoAnalyseCompilation());
         }
 
-        private static bool FileIsUpToDate(string src, string dest)
-        {
-            return File.Exists(dest) &&
-                File.GetLastWriteTime(dest) >= File.GetLastWriteTime(src);
-        }
-
         private static void AnalyseNamespace(Context cx, INamespaceSymbol ns)
         {
             foreach (var memberNamespace in ns.GetNamespaceMembers())

csharp/extractor/Semmle.Extraction.Tests/Assets.cs

@@ -16,7 +16,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsJson1;
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.True(success);
@@ -46,7 +46,7 @@ namespace Semmle.Extraction.Tests
             var json = "garbage data";
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.False(success);
@@ -61,7 +61,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNet70;
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.True(success);
@@ -91,7 +91,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNet48;
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.True(success);
@@ -117,7 +117,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNetstandard21;
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.True(success);
@@ -144,7 +144,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNetstandard16;
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.True(success);
@@ -175,7 +175,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNetcoreapp20;
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.True(success);
@@ -202,7 +202,7 @@ namespace Semmle.Extraction.Tests
             var json = assetsNetcoreapp31;
 
             // Execute
-            var success = assets.TryParse(json);
+            var success = assets.TryParse(json, "");
 
             // Verify
             Assert.True(success);

csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs

@@ -10,27 +10,36 @@ namespace Semmle.Extraction.Entities
         private static int messageCount = 0;
 
         private readonly Message msg;
+        private readonly bool bypassLimit;
 
-        public ExtractionMessage(Context cx, Message msg) : base(cx)
+        public ExtractionMessage(Context cx, Message msg) : this(cx, msg, bypassLimit: false)
         {
+        }
+
+        private ExtractionMessage(Context cx, Message msg, bool bypassLimit) : base(cx)
+        {
+            this.bypassLimit = bypassLimit;
             this.msg = msg;
             TryPopulate();
         }
 
         protected override void Populate(TextWriter trapFile)
         {
-            // The below doesn't limit the extractor messages to the exact limit, but it's good enough.
-            Interlocked.Increment(ref messageCount);
-            if (messageCount > limit)
+            if (!bypassLimit)
             {
-                if (messageCount == limit + 1)
+                var val = Interlocked.Increment(ref messageCount);
+                if (val > limit)
                 {
-                    Context.ExtractionContext.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}");
+                    if (val == limit + 1)
+                    {
+                        Context.ExtractionContext.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}");
+                        _ = new ExtractionMessage(Context, new Message($"Stopped logging extractor messages after reaching {limit}", null, null, null, Util.Logging.Severity.Warning), bypassLimit: true);
+                    }
+                    return;
                 }
-                return;
             }
 
-            trapFile.extractor_messages(this, msg.Severity, "C# extractor", msg.Text, msg.EntityText ?? string.Empty,
+            trapFile.extractor_messages(this, msg.Severity, msg.Text, msg.EntityText ?? string.Empty,
                 msg.Location ?? Context.CreateLocation(), msg.StackTrace ?? string.Empty);
         }
     }

csharp/extractor/Semmle.Extraction/TrapWriter.cs

@@ -194,8 +194,11 @@ namespace Semmle.Extraction
                     var hash = FileUtils.ComputeFileHash(tmpFile);
                     if (existingHash != hash)
                     {
-                        var root = TrapFile.Substring(0, TrapFile.Length - 8); // Remove trailing ".trap.gz"
-                        if (TryMove(tmpFile, $"{root}-{hash}.trap{TrapExtension(trapCompression)}"))
+                        var extension = TrapExtension(trapCompression);
+                        var root = TrapFile[..^extension.Length]; // Remove trailing ".trap", ".trap.gz", or ".trap.br"
+                        var newTrapName = $"{root}-{hash}{extension}";
+                        logger.LogInfo($"Identical trap file for {TrapFile} already exists, renaming to {newTrapName}");
+                        if (TryMove(tmpFile, $"{newTrapName}"))
                             return;
                     }
                     logger.LogInfo($"Identical trap file for {TrapFile} already exists");
@@ -217,16 +220,16 @@ namespace Semmle.Extraction
         {
             switch (compression)
             {
-                case CompressionMode.None: return "";
-                case CompressionMode.Gzip: return ".gz";
-                case CompressionMode.Brotli: return ".br";
+                case CompressionMode.None: return ".trap";
+                case CompressionMode.Gzip: return ".trap.gz";
+                case CompressionMode.Brotli: return ".trap.br";
                 default: throw new ArgumentOutOfRangeException(nameof(compression), compression, "Unsupported compression type");
             }
         }
 
         public static string TrapPath(ILogger logger, string? folder, PathTransformer.ITransformedPath path, TrapWriter.CompressionMode trapCompression)
         {
-            var filename = $"{path.Value}.trap{TrapExtension(trapCompression)}";
+            var filename = $"{path.Value}{TrapExtension(trapCompression)}";
             if (string.IsNullOrEmpty(folder))
                 folder = Directory.GetCurrentDirectory();
 

csharp/extractor/Semmle.Extraction/Tuples.cs

@@ -12,9 +12,9 @@ namespace Semmle.Extraction
             trapFile.WriteTuple("containerparent", parent, child);
         }
 
-        internal static void extractor_messages(this System.IO.TextWriter trapFile, ExtractionMessage error, Semmle.Util.Logging.Severity severity, string origin, string errorMessage, string entityText, Location location, string stackTrace)
+        internal static void extractor_messages(this System.IO.TextWriter trapFile, ExtractionMessage error, Semmle.Util.Logging.Severity severity, string errorMessage, string entityText, Location location, string stackTrace)
         {
-            trapFile.WriteTuple("extractor_messages", error, (int)severity, origin, errorMessage, entityText, location, stackTrace);
+            trapFile.WriteTuple("extractor_messages", error, (int)severity, "C# extractor", errorMessage, entityText, location, stackTrace);
         }
 
         public static void files(this System.IO.TextWriter trapFile, File file, string fullName)

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.7.22
+version: 1.7.23-dev
 groups:
   - csharp
   - solorigate

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.7.22
+version: 1.7.23-dev
 groups:
   - csharp
   - solorigate

csharp/ql/integration-tests/all-platforms/autobuild/test.py

@@ -1,3 +1,2 @@
-from create_database_utils import *
-
-run_codeql_database_create([], lang="csharp")
+def test(codeql, csharp):
+    codeql.database.create()

csharp/ql/integration-tests/all-platforms/binlog/test.py

@@ -1,7 +1,6 @@
-import subprocess
-from create_database_utils import *
-from diagnostics_test_utils import *
+import commands
 
-subprocess.check_call(["dotnet", "build", "test.sln", "/bl:test.binlog"])
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none", "-Obinlog=test.binlog"])
-check_diagnostics()
+
+def test(codeql, csharp):
+    commands.run(["dotnet", "build", "test.sln", "/bl:test.binlog"])
+    codeql.database.create(build_mode="none", extractor_option="binlog=test.binlog")

csharp/ql/integration-tests/all-platforms/conditional_compilation/Methods.expected

@@ -0,0 +1,4 @@
+| Program.cs:3:24:3:27 | Main | 2 |
+| Program.cs:10:17:10:33 | GreetConditional1 | 2 |
+| Program.cs:19:17:19:21 | Greet | 2 |
+| Program.cs:25:17:25:33 | GreetConditional2 | 2 |

csharp/ql/integration-tests/all-platforms/conditional_compilation/Methods.ql

@@ -0,0 +1,5 @@
+import csharp
+
+from Method m
+where m.fromSource()
+select m, count(m.getBody())

csharp/ql/integration-tests/all-platforms/conditional_compilation/Program.cs

@@ -0,0 +1,40 @@
+public class Test
+{
+    public static void Main()
+    {
+        Greet();
+        GreetConditional1();
+        GreetConditional2();
+    }
+
+    static void GreetConditional1()
+    {
+#if A
+    Console.WriteLine("Hello, A!");
+#elif C
+    Console.WriteLine("Hello, C!");
+#endif
+    }
+
+    static void Greet()
+    {
+        Console.WriteLine("Hello, World!");
+    }
+
+#if A
+    static void GreetConditional2()
+    {
+        Console.WriteLine("Hello, A!");
+    }
+#elif B
+    static void GreetConditional2()
+    {
+        Console.WriteLine("Hello, B!");
+    }
+#else
+    static void GreetConditional2()
+    {
+        Console.WriteLine("Hello, Others!");
+    }
+#endif
+}

csharp/ql/integration-tests/all-platforms/conditional_compilation/global.json

@@ -0,0 +1,5 @@
+{
+    "sdk": {
+        "version": "8.0.101"
+    }
+}

csharp/ql/integration-tests/all-platforms/conditional_compilation/test.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Remove="**/*.cs" />
+    <Compile Include="Program.cs" />
+  </ItemGroup>
+
+</Project>

csharp/ql/integration-tests/all-platforms/conditional_compilation/test.py

@@ -0,0 +1,8 @@
+import os
+
+
+def test(codeql, csharp):
+    os.environ["CODEQL_EXTRACTOR_CSHARP_OPTION_TRAP_COMPRESSION"] = "none"
+    codeql.database.create(
+        command=["dotnet build /p:DefineConstants=A", "dotnet build /p:DefineConstants=B"]
+    )

csharp/ql/integration-tests/all-platforms/cshtml/test.py

@@ -1,3 +1,2 @@
-from create_database_utils import *
-
-run_codeql_database_create(['dotnet build'], lang="csharp")
+def test(codeql, csharp):
+    codeql.database.create(command="dotnet build")

csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.expected

@@ -1,4 +1,4 @@
 | Program.cs |
 | Views/Home/Index.cshtml |
-| _semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_Views_Home_Index_cshtml.g.cs |
+| _ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_test_test_Views_Home_Index_cshtml.g.cs |
 | test-db/working/implicitUsings/GlobalUsings.g.cs |

csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.ql

@@ -2,15 +2,12 @@ import csharp
 
 private string getPath(File f) {
   result = f.getRelativePath() and
-  not exists(
-    result
-        .indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_")
-  )
+  not exists(result.indexOf("_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_"))
   or
   exists(int index |
     index =
       f.getRelativePath()
-          .indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_") and
+          .indexOf("_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_") and
     result = f.getRelativePath().substring(index, f.getRelativePath().length())
   )
 }

csharp/ql/integration-tests/all-platforms/cshtml_standalone/test.py

@@ -1,4 +1,2 @@
-import os
-from create_database_utils import *
-
-run_codeql_database_create(lang="csharp", extra_args=["--build-mode=none"])
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/all-platforms/cshtml_standalone_disabled/test.py

@@ -1,5 +1,6 @@
 import os
-from create_database_utils import *
 
-os.environ['CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_WEB_VIEWS'] = 'false'
-run_codeql_database_create(lang="csharp", extra_args=["--build-mode=none"])
+
+def test(codeql, csharp):
+    os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_WEB_VIEWS"] = "false"
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/all-platforms/cshtml_standalone_flowsteps/test.py

@@ -1,4 +1,2 @@
-import os
-from create_database_utils import *
-
-run_codeql_database_create(lang="csharp", extra_args=["--build-mode=none"])
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/all-platforms/cshtml_standalone_net6/Files.expected

@@ -1,4 +1,4 @@
 | Program.cs |
 | Views/Home/Index.cshtml |
-| _semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_net6_Views_Home_Index_cshtml.g.cs |
+| _ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_net6_test_test_Views_Home_Index_cshtml.g.cs |
 | test-db/working/implicitUsings/GlobalUsings.g.cs |

csharp/ql/integration-tests/all-platforms/cshtml_standalone_net6/Files.ql

@@ -2,15 +2,12 @@ import csharp
 
 private string getPath(File f) {
   result = f.getRelativePath() and
-  not exists(
-    result
-        .indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_")
-  )
+  not exists(result.indexOf("_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_"))
   or
   exists(int index |
     index =
       f.getRelativePath()
-          .indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_") and
+          .indexOf("_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_") and
     result = f.getRelativePath().substring(index, f.getRelativePath().length())
   )
 }

csharp/ql/integration-tests/all-platforms/cshtml_standalone_net6/test.py

@@ -1,4 +1,2 @@
-import os
-from create_database_utils import *
-
-run_codeql_database_create(lang="csharp", extra_args=["--build-mode=none"])
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/all-platforms/diag_dotnet_incompatible/test.py

@@ -1,5 +1,2 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
-run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
-check_diagnostics()
+def test(codeql, csharp):
+    codeql.database.create(_assert_failure=True)

csharp/ql/integration-tests/all-platforms/diag_missing_project_files/test.py

@@ -1,5 +1,2 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
-run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
-check_diagnostics()
+def test(codeql, csharp):
+    codeql.database.create(_assert_failure=True)

csharp/ql/integration-tests/all-platforms/diag_missing_xamarin_sdk/test.py

@@ -1,5 +1,2 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
-run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
-check_diagnostics()
+def test(codeql, csharp):
+    codeql.database.create(_assert_failure=True)

csharp/ql/integration-tests/all-platforms/diag_recursive_generics/test.py

@@ -1,3 +1,2 @@
-from create_database_utils import *
-
-run_codeql_database_create(['dotnet build'], lang="csharp")
+def test(codeql, csharp):
+    codeql.database.create(command="dotnet build")

csharp/ql/integration-tests/all-platforms/dotnet_build/test.py

@@ -1,16 +1,21 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
 def check_build_out(msg, s):
-    if "[build-stdout] " + msg not in s:
-        raise Exception("The C# tracer did not interpret the dotnet path-to-application command correctly.")
+    assert (
+        "[build-stdout] " + msg in s
+    ), f"The C# tracer did not interpret the dotnet path-to-application command correctly."
+
+
+def test1(codeql, csharp):
+    codeql.database.create(command="dotnet build")
 
-run_codeql_database_create(['dotnet build'], test_db="test1-db", lang="csharp")
-check_diagnostics(test_db="test1-db")
 
 # This test checks that we don't inject any flags when running the application using `dotnet`
-my_dir = "my_program"
-my_abs_path = os.path.abspath(f"{my_dir}/dotnet_build.dll")
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test1-db', 'dotnet build -o my_program', f'dotnet {my_abs_path} build is not a subcommand'], "test2-db", "csharp")
-check_build_out("<arguments>build,is,not,a,subcommand</arguments>", s)
-check_diagnostics(test_db="test2-db")
+def test2(codeql, csharp, cwd):
+    s = codeql.database.create(
+        command=[
+            "dotnet build -o my_program",
+            f"dotnet {cwd / 'my_program'}/dotnet_build.dll build is not a subcommand",
+        ],
+        _capture="stdout",
+    )
+
+    check_build_out("<arguments>build,is,not,a,subcommand</arguments>", s)

csharp/ql/integration-tests/all-platforms/dotnet_no_args_inject/test.py

@@ -1,9 +1,9 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
-# the tracer configuration should not inject the extra command-line arguments for these commands
-# and they should therefore run successfully
-run_codeql_database_init(lang="csharp")
-# this command fails on Windows for some reason, so we comment it out for now
-# run_codeql_database_trace_command(['dotnet', 'tool', 'search', 'publish'])
-run_codeql_database_trace_command(['dotnet', 'new', 'console', '--force', '--name', 'build', '--output', '.'])
+def test(codeql, csharp):
+    codeql.database.init("test-db", source_root=".")
+    # the tracer configuration should not inject the extra command-line arguments for these commands
+    # and they should therefore run successfully
+    # this command fails on Windows for some reason, so we comment it out for now
+    # run_codeql_database_trace_command(['dotnet', 'tool', 'search', 'publish'])
+    codeql.database.trace_command(
+        "test-db", "dotnet", "new", "console", "--force", "--name", "build", "--output", "."
+    )

csharp/ql/integration-tests/all-platforms/dotnet_pack/test.py

@@ -1,11 +1,8 @@
 import os
-from create_database_utils import *
-from diagnostics_test_utils import *
 
-run_codeql_database_create(['dotnet pack -o nugetpackage'], db=None, lang="csharp")
 
-## Check that the NuGet package is created.
-if not os.path.isfile("nugetpackage/dotnet_pack.1.0.0.nupkg"):
-    raise Exception("The NuGet package was not created.")
-
-check_diagnostics()
+def test(codeql, csharp):
+    codeql.database.create(command="dotnet pack -o nugetpackage")
+    assert os.path.isfile(
+        "nugetpackage/dotnet_pack.1.0.0.nupkg"
+    ), "The NuGet package was not created."

csharp/ql/integration-tests/all-platforms/dotnet_publish/test.py

@@ -1,12 +1,7 @@
 import os
-from create_database_utils import *
-from diagnostics_test_utils import *
 
-artifacts = 'bin/Temp'
-run_codeql_database_create([f"dotnet publish -o {artifacts}"], db=None, lang="csharp")
 
-## Check that the publish folder is created.
-if not os.path.isdir(artifacts):
-    raise Exception("The publish artifact folder was not created.")
-
-check_diagnostics()
+def test(codeql, csharp):
+    artifacts = "bin/Temp"
+    codeql.database.create(command=f"dotnet publish -o {artifacts}")
+    assert os.path.isdir(artifacts), "The publish artifact folder was not created."

csharp/ql/integration-tests/all-platforms/dotnet_run/test.py

@@ -1,57 +1,69 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
 def check_build_out(msg, s):
-    if "[build-stdout] " + msg not in s:
-        raise Exception("The C# tracer did not interpret the 'dotnet run' command correctly")
+    assert (
+        "[build-stdout] " + msg in s
+    ), "The C# tracer did not interpret the 'dotnet run' command correctly"
+
 
 # no arguments
-s = run_codeql_database_create_stdout(['dotnet run'], "test-db", "csharp")
-check_build_out("Default reply", s)
-check_diagnostics()
+def test_no_args(codeql, csharp):
+    s = codeql.database.create(command="dotnet run", _capture="stdout")
+    check_build_out("Default reply", s)
+
 
 # no arguments, but `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test-db', 'dotnet run --'], "test2-db", "csharp")
-check_build_out("Default reply", s)
-check_diagnostics(test_db="test2-db")
+def test_no_arg_dash_dash(codeql, csharp):
+    s = codeql.database.create(command="dotnet run --", _capture="stdout")
+    check_build_out("Default reply", s)
+
 
 # one argument, no `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test2-db', 'dotnet run hello'], "test3-db", "csharp")
-check_build_out("Default reply", s)
-check_diagnostics(test_db="test3-db")
+def test_one_arg_no_dash_dash(codeql, csharp):
+    s = codeql.database.create(command="dotnet run hello", _capture="stdout")
+    check_build_out("Default reply", s)
+
 
 # one argument, but `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test3-db', 'dotnet run -- hello'], "test4-db", "csharp")
-check_build_out("Default reply", s)
-check_diagnostics(test_db="test4-db")
+def test_one_arg_dash_dash(codeql, csharp):
+    s = codeql.database.create(command="dotnet run -- hello", _capture="stdout")
+    check_build_out("Default reply", s)
+
 
 # two arguments, no `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test4-db', 'dotnet run hello world'], "test5-db", "csharp")
-check_build_out("hello, world", s)
-check_diagnostics(test_db="test5-db")
+def test_two_args_no_dash_dash(codeql, csharp):
+    s = codeql.database.create(command="dotnet run hello world", _capture="stdout")
+    check_build_out("hello, world", s)
+
 
 # two arguments, and `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test5-db', 'dotnet run -- hello world'], "test6-db", "csharp")
-check_build_out("hello, world", s)
-check_diagnostics(test_db="test6-db")
+def test_two_args_dash_dash(codeql, csharp):
+    s = codeql.database.create(command="dotnet run -- hello world", _capture="stdout")
+    check_build_out("hello, world", s)
+
 
 # shared compilation enabled; tracer should override by changing the command
 # to `dotnet run -p:UseSharedCompilation=true -p:UseSharedCompilation=false -- hello world`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test6-db', 'dotnet run -p:UseSharedCompilation=true -- hello world'], "test7-db", "csharp")
-check_build_out("hello, world", s)
-check_diagnostics(test_db="test7-db")
+def test_shared_compilation(codeql, csharp):
+    s = codeql.database.create(
+        command="dotnet run -p:UseSharedCompilation=true -- hello world", _capture="stdout"
+    )
+    check_build_out("hello, world", s)
+
 
 # option passed into `dotnet run`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test7-db', 'dotnet build', 'dotnet run --no-build hello world'], "test8-db", "csharp")
-check_build_out("hello, world", s)
-check_diagnostics(test_db="test8-db")
+def test_option(codeql, csharp):
+    s = codeql.database.create(
+        command=["dotnet build", "dotnet run --no-build hello world"], _capture="stdout"
+    )
+    check_build_out("hello, world", s)
+
 
 # two arguments, no '--' (first argument quoted)
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test8-db', 'dotnet run "hello world part1" part2'], "test9-db", "csharp")
-check_build_out("hello world part1, part2", s)
-check_diagnostics(test_db="test9-db")
+def test_two_args_no_dash_dash_quote_first(codeql, csharp):
+    s = codeql.database.create(command='dotnet run "hello world" part2', _capture="stdout")
+    check_build_out("hello world, part2", s)
+
 
 # two arguments, no '--' (second argument quoted) and using dotnet to execute dotnet
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test9-db', 'dotnet dotnet run part1 "hello world part2"'], "test10-db", "csharp")
-check_build_out("part1, hello world part2", s)
-check_diagnostics(test_db="test10-db")
+def test_two_args_no_dash_dash_quote_second(codeql, csharp):
+    s = codeql.database.create(command='dotnet dotnet run hello "world part2"', _capture="stdout")
+    check_build_out("hello, world part2", s)

csharp/ql/integration-tests/all-platforms/msbuild/test.py

@@ -1,6 +1,3 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
-# force CodeQL to use MSBuild by setting `LGTM_INDEX_MSBUILD_TARGET`
-run_codeql_database_create([], db=None, lang="csharp", extra_env={ 'LGTM_INDEX_MSBUILD_TARGET': 'Build' })
-check_diagnostics()
+def test(codeql, csharp):
+    # force CodeQL to use MSBuild by setting `LGTM_INDEX_MSBUILD_TARGET`
+    codeql.database.create(_env={"LGTM_INDEX_MSBUILD_TARGET": "Build"})

csharp/ql/integration-tests/all-platforms/source_generator/test.py

@@ -1,3 +1,2 @@
-from create_database_utils import *
-
-run_codeql_database_create(['dotnet build'], lang="csharp")
+def test(codeql, csharp):
+    codeql.database.create(command="dotnet build")

csharp/ql/integration-tests/all-platforms/standalone/DatabaseQualityDiagnostics.expected

@@ -0,0 +1,6 @@
+diagnosticAttributes
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityCliSummaryTable | true |
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityStatusPage | true |
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityTelemetry | true |
+#select
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | 1 |

csharp/ql/integration-tests/all-platforms/standalone/DatabaseQualityDiagnostics.qlref

@@ -0,0 +1 @@
+Telemetry/DatabaseQualityDiagnostics.ql

csharp/ql/integration-tests/all-platforms/standalone/Diag.expected

@@ -1,7 +1,9 @@
 extractorMessages
-| 5 |
+| 6 |
 compilerDiagnostics
 | 4 |
+extractorMessagesLeachedLimit
+| Program.cs:1:1:1:0 | Stopped logging extractor messages after reaching 5 |
 compilationInfo
 | Compiler diagnostic count for CS0103 | 3.0 |
 | Compiler diagnostic count for CS8019 | 7.0 |

csharp/ql/integration-tests/all-platforms/standalone/Diag.ql

@@ -5,6 +5,10 @@ query predicate extractorMessages(int c) { c = count(ExtractorMessage msg) }
 
 query predicate compilerDiagnostics(int c) { c = count(Diagnostic diag) }
 
+query predicate extractorMessagesLeachedLimit(ExtractorMessage msg) {
+  msg.getText().indexOf("Stopped logging") = 0
+}
+
 query predicate compilationInfo(string key, float value) {
   exists(Compilation c, string infoValue |
     infoValue = c.getInfo(key) and key.matches("Compiler diagnostic count for%")

csharp/ql/integration-tests/all-platforms/standalone/test.py

@@ -1,9 +1,7 @@
 import os
-from create_database_utils import *
-from diagnostics_test_utils import *
 
-os.environ['CODEQL_EXTRACTOR_CSHARP_OPTION_COMPILER_DIAGNOSTIC_LIMIT'] = '2'
-os.environ['CODEQL_EXTRACTOR_CSHARP_OPTION_MESSAGE_LIMIT'] = '5'
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
 
-check_diagnostics()
+def test(codeql, csharp):
+    os.environ["CODEQL_EXTRACTOR_CSHARP_OPTION_COMPILER_DIAGNOSTIC_LIMIT"] = "2"
+    os.environ["CODEQL_EXTRACTOR_CSHARP_OPTION_MESSAGE_LIMIT"] = "5"
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/all-platforms/standalone_buildless_option/DatabaseQualityDiagnostics.expected

@@ -0,0 +1,2 @@
+diagnosticAttributes
+#select

csharp/ql/integration-tests/all-platforms/standalone_buildless_option/DatabaseQualityDiagnostics.qlref

@@ -0,0 +1 @@
+Telemetry/DatabaseQualityDiagnostics.ql

csharp/ql/integration-tests/all-platforms/standalone_buildless_option/test.py

@@ -1,8 +1,6 @@
 import os
-from create_database_utils import *
-from diagnostics_test_utils import *
 
-os.environ['CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS'] = 'true'
-run_codeql_database_create([], lang="csharp")
 
-check_diagnostics()
+def test(codeql, csharp):
+    os.environ["CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS"] = "true"
+    codeql.database.create()

csharp/ql/integration-tests/all-platforms/standalone_dependencies_net48/test.py

@@ -1,3 +1,2 @@
-from create_database_utils import *
-
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/all-platforms/standalone_failed/test.py

@@ -1,6 +1,2 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
-
-run_codeql_database_create([], db=None, lang="csharp", extra_args=["--build-mode=none"], runFunction=runUnsuccessfully)
-
-check_diagnostics()
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none", _assert_failure=True)

csharp/ql/integration-tests/all-platforms/standalone_resx/test.py

@@ -1,6 +1,6 @@
 import os
-from create_database_utils import *
 
-os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_RESOURCES"] = "true"
 
-run_codeql_database_create(lang="csharp", extra_args=["--build-mode=none"])
+def test(codeql, csharp):
+    os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_RESOURCES"] = "true"
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/all-platforms/standalone_winforms/test.py

@@ -1,3 +1,2 @@
-from create_database_utils import *
-
-run_codeql_database_create(lang="csharp", extra_args=["--build-mode=none"])
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/legacy

@@ -1 +0,0 @@
-These tests are still run with the legacy test runner

csharp/ql/integration-tests/linux-only/compiler_args/CompilerArgs.expected

@@ -183,25 +183,19 @@
 | 182 | /warnaserror- |
 | 183 | /utf8output |
 | 184 | /deterministic+ |
-| 185 | /sourcelink:obj/Debug/net8.0/test.sourcelink.json |
-| 186 | /langversion:12.0 |
-| 187 | /embed:Program.cs |
-| 188 | /embed:obj/Debug/net8.0/test.GlobalUsings.g.cs |
-| 189 | /embed:"obj/Debug/net8.0/.NETCoreApp,Version=v8.0.AssemblyAttributes.cs" |
-| 190 | /embed:obj/Debug/net8.0/test.AssemblyInfo.cs |
-| 191 | /analyzerconfig:/home/runner/work/semmle-code/semmle-code/.editorconfig |
-| 192 | /analyzerconfig:obj/Debug/net8.0/test.GeneratedMSBuildEditorConfig.editorconfig |
-| 193 | /analyzerconfig:[...]/8.0.101/Sdks/Microsoft.NET.Sdk/analyzers/build/config/analysislevel_8_default.globalconfig |
-| 194 | /analyzer:[...]/8.0.101/Sdks/Microsoft.NET.Sdk/targets/../analyzers/Microsoft.CodeAnalysis.CSharp.NetAnalyzers.dll |
-| 195 | /analyzer:[...]/8.0.101/Sdks/Microsoft.NET.Sdk/targets/../analyzers/Microsoft.CodeAnalysis.NetAnalyzers.dll |
-| 196 | /analyzer:[...]/8.0.1/analyzers/dotnet/cs/Microsoft.Interop.ComInterfaceGenerator.dll |
-| 197 | /analyzer:[...]/8.0.1/analyzers/dotnet/cs/Microsoft.Interop.JavaScript.JSImportGenerator.dll |
-| 198 | /analyzer:[...]/8.0.1/analyzers/dotnet/cs/Microsoft.Interop.LibraryImportGenerator.dll |
-| 199 | /analyzer:[...]/8.0.1/analyzers/dotnet/cs/Microsoft.Interop.SourceGeneration.dll |
-| 200 | /analyzer:[...]/8.0.1/analyzers/dotnet/cs/System.Text.Json.SourceGeneration.dll |
-| 201 | /analyzer:[...]/8.0.1/analyzers/dotnet/cs/System.Text.RegularExpressions.Generator.dll |
-| 202 | Program.cs |
-| 203 | obj/Debug/net8.0/test.GlobalUsings.g.cs |
-| 204 | obj/Debug/net8.0/.NETCoreApp,Version=v8.0.AssemblyAttributes.cs |
-| 205 | obj/Debug/net8.0/test.AssemblyInfo.cs |
-| 206 | /warnaserror+:NU1605,SYSLIB0011 |
+| 185 | /langversion:12.0 |
+| 186 | /analyzerconfig:obj/Debug/net8.0/test.GeneratedMSBuildEditorConfig.editorconfig |
+| 187 | /analyzerconfig:[...]/8.0.101/Sdks/Microsoft.NET.Sdk/analyzers/build/config/analysislevel_8_default.globalconfig |
+| 188 | /analyzer:[...]/8.0.101/Sdks/Microsoft.NET.Sdk/targets/../analyzers/Microsoft.CodeAnalysis.CSharp.NetAnalyzers.dll |
+| 189 | /analyzer:[...]/8.0.101/Sdks/Microsoft.NET.Sdk/targets/../analyzers/Microsoft.CodeAnalysis.NetAnalyzers.dll |
+| 190 | /analyzer:[...]/8.0.1/analyzers/dotnet/cs/Microsoft.Interop.ComInterfaceGenerator.dll |
+| 191 | /analyzer:[...]/8.0.1/analyzers/dotnet/cs/Microsoft.Interop.JavaScript.JSImportGenerator.dll |
+| 192 | /analyzer:[...]/8.0.1/analyzers/dotnet/cs/Microsoft.Interop.LibraryImportGenerator.dll |
+| 193 | /analyzer:[...]/8.0.1/analyzers/dotnet/cs/Microsoft.Interop.SourceGeneration.dll |
+| 194 | /analyzer:[...]/8.0.1/analyzers/dotnet/cs/System.Text.Json.SourceGeneration.dll |
+| 195 | /analyzer:[...]/8.0.1/analyzers/dotnet/cs/System.Text.RegularExpressions.Generator.dll |
+| 196 | Program.cs |
+| 197 | obj/Debug/net8.0/test.GlobalUsings.g.cs |
+| 198 | obj/Debug/net8.0/.NETCoreApp,Version=v8.0.AssemblyAttributes.cs |
+| 199 | obj/Debug/net8.0/test.AssemblyInfo.cs |
+| 200 | /warnaserror+:NU1605,SYSLIB0011 |

csharp/ql/integration-tests/linux-only/compiler_args/test.py

@@ -1,3 +1,6 @@
-from create_database_utils import *
+import runs_on
 
-run_codeql_database_create([], lang="csharp")
+
+@runs_on.linux
+def test(codeql, csharp):
+    codeql.database.create()

csharp/ql/integration-tests/linux-only/standalone_dependencies_non_utf8_filename/test.py

@@ -1,8 +1,10 @@
-from create_database_utils import *
+import runs_on
 
-path = b'\xd2abcd.cs'
 
-with open(path, 'w') as file:
-  file.write('class X { }\n')
+@runs_on.linux
+def test(codeql, csharp):
+    path = b"\xd2abcd.cs"
 
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+    with open(path, "w") as file:
+        file.write("class X { }\n")
+        codeql.database.create(build_mode="none")

csharp/ql/integration-tests/posix-only/diag_autobuild_script/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.posix
+def test(codeql, csharp):
+    codeql.database.create(_assert_failure=True)

csharp/ql/integration-tests/posix-only/diag_multiple_scripts/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.posix
+def test(codeql, csharp):
+    codeql.database.create(_assert_failure=True)

csharp/ql/integration-tests/posix-only/dotnet_test/test.py

@@ -1,15 +1,19 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-# Implicitly build and then run tests.
-run_codeql_database_create(['dotnet test'], test_db="test-db", lang="csharp")
-check_diagnostics()
 
-# Explicitly build and then run tests.
-run_codeql_database_create(['dotnet clean', 'rm -rf test-db', 'dotnet build -o myout', 'dotnet test myout/dotnet_test.dll'], test_db="test2-db", lang="csharp")
-check_diagnostics(test_db="test2-db")
+@runs_on.posix
+def test_implicit_build_then_run(codeql, csharp):
+    codeql.database.create(command="dotnet test")
+
+
+@runs_on.posix
+def test_explicit_build_then_run(codeql, csharp):
+    codeql.database.create(command=["dotnet build -o myout", "dotnet test myout/dotnet_test.dll"])
+
 
-thisDir = os.path.abspath(os.getcwd())
 # Explicit build and then run tests using the absolute path.
-run_codeql_database_create(['dotnet clean', 'rm -rf test2-db', 'dotnet build -o myout', f'dotnet test {thisDir}/myout/dotnet_test.dll'], test_db="test3-db", lang="csharp")
-check_diagnostics(test_db="test3-db")
+@runs_on.posix
+def test_explicit_build_then_run_abs_path(codeql, csharp, cwd):
+    codeql.database.create(
+        command=["dotnet build -o myout", f"dotnet test {cwd}/myout/dotnet_test.dll"]
+    )

csharp/ql/integration-tests/posix-only/dotnet_test_mstest/test.py

@@ -1,14 +1,19 @@
-import platform
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-# Implicitly build and then run tests.
-run_codeql_database_create(['dotnet test'], test_db="test-db", lang="csharp")
-check_diagnostics()
 
-# Fix `dotnet test` picking `x64` on arm-based macOS
-architecture = '-a arm64' if platform.machine() == 'arm64' else ''
+@runs_on.posix
+def test_implicit_build_and_test(codeql, csharp):
+    codeql.database.create(command="dotnet test")
+
 
 # Explicitly build and then run tests.
-run_codeql_database_create(['dotnet clean', 'rm -rf test-db', 'dotnet build -o myout --os win', 'dotnet test myout/dotnet_test_mstest.exe ' + architecture], test_db="test2-db", lang="csharp")
-check_diagnostics(test_db="test2-db")
+@runs_on.posix
+def test_explicit_build_and_test(codeql, csharp):
+    # Fix `dotnet test` picking `x64` on arm-based macOS
+    architecture = "-a arm64" if runs_on.arm64 else ""
+    codeql.database.create(
+        command=[
+            "dotnet build -o myout --os win",
+            f"dotnet test myout/dotnet_test_mstest.exe {architecture}",
+        ]
+    )

csharp/ql/integration-tests/posix-only/inherit-env-vars/test.py

@@ -1,8 +1,8 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
 import os
+import runs_on
 
-os.environ["PROJECT_TO_BUILD"] = "proj.csproj.no_auto"
 
-run_codeql_database_create([], db=None, lang="csharp")
-check_diagnostics()
+@runs_on.posix
+def test(codeql, csharp):
+    os.environ["PROJECT_TO_BUILD"] = "proj.csproj.no_auto"
+    codeql.database.create()

csharp/ql/integration-tests/posix-only/standalone_dependencies/test.py

@@ -1,3 +1,6 @@
-from create_database_utils import *
+import runs_on
 
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+
+@runs_on.posix
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/test.py

@@ -1,3 +1,6 @@
-from create_database_utils import *
+import runs_on
 
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+
+@runs_on.posix
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/posix-only/standalone_dependencies_multi_project/test.py

@@ -1,3 +1,6 @@
-from create_database_utils import *
+import runs_on
 
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+
+@runs_on.posix
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/posix-only/standalone_dependencies_multi_target/test.py

@@ -1,3 +1,6 @@
-from create_database_utils import *
+import runs_on
 
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+
+@runs_on.posix
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/skip-on-platform-osx-arm

@@ -1 +0,0 @@
-Skipping the test on the ARM runners, as we're running into trouble with Mono and nuget.

csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/test.py

@@ -1,5 +1,12 @@
-from create_database_utils import *
+import runs_on
+import pytest
 import os
 
-os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES"] = "/non-existent-path"
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+
+# Skipping the test on the ARM runners, as we're running into trouble with Mono and nuget.
+@pytest.mark.only_if(runs_on.linux or (runs_on.macos and runs_on.x86_64))
+def test(codeql, csharp):
+    os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES"] = (
+        "/non-existent-path"
+    )
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget with_space/skip-on-platform-osx-arm

@@ -1 +0,0 @@
-Skipping the test on the ARM runners, as we're running into trouble with Mono and nuget.

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget with_space/test.py

@@ -1,8 +1,12 @@
-from create_database_utils import *
 import os
+import runs_on
+import pytest
 
-# making sure we're not doing any fallback restore:
-os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT"] = "1"
-os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT"] = "1"
 
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+# Skipping the test on the ARM runners, as we're running into trouble with Mono and nuget.
+@pytest.mark.only_if(runs_on.linux or (runs_on.macos and runs_on.x86_64))
+def test(codeql, csharp):
+    # making sure we're not doing any fallback restore:
+    os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT"] = "1"
+    os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT"] = "1"
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget/skip-on-platform-osx-arm

@@ -1 +0,0 @@
-Skipping the test on the ARM runners, as we're running into trouble with Mono and nuget.

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget/test.py

@@ -1,3 +1,8 @@
-from create_database_utils import *
+import runs_on
+import pytest
 
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+
+# Skipping the test on the ARM runners, as we're running into trouble with Mono and nuget.
+@pytest.mark.only_if(runs_on.linux or (runs_on.macos and runs_on.x86_64))
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/test.py

@@ -1,3 +1,6 @@
-from create_database_utils import *
+import runs_on
 
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+
+@runs_on.posix
+def test(codeql, csharp):
+    codeql.database.create(build_mode="none")

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/diagnostics.expected

@@ -28,7 +28,7 @@
 }
 {
   "markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
-  "severity": "warning",
+  "severity": "note",
   "source": {
     "extractorName": "csharp",
     "id": "csharp/autobuilder/buildless/unreachable-feed",

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/test.py

@@ -1,17 +1,26 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
 import os
+import runs_on
 
-# os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK"] = "true"        # Nuget feed check is enabled by default
-os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT"] = "1"     # 1ms, the GET request should fail with such short timeout
-os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT"] = "1"       # Limit the count of checks to 1
-os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED"] = "https://abc.de:8000/packages/"   # Exclude this feed from check
 
-# Making sure the reachability test of `nuget.org` succeeds:
-os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT"] = "1000"
-os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT"] = "5"
-# The second feed is ignored in the fallback restore, because of network issues:
-os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK"] = "https://api.nuget.org/v3/index.json https://abc.def:8000/packages/"
+@runs_on.posix
+def test(codeql, csharp):
+    # os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK"] = "true"        # Nuget feed check is enabled by default
+    os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT"] = (
+        "1"  # 1ms, the GET request should fail with such short timeout
+    )
+    os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT"] = (
+        "1"  # Limit the count of checks to 1
+    )
+    os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED"] = (
+        "https://abc.de:8000/packages/"  # Exclude this feed from check
+    )
 
-run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
-check_diagnostics()
+    # Making sure the reachability test of `nuget.org` succeeds:
+    os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT"] = "1000"
+    os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT"] = "5"
+    # The second feed is ignored in the fallback restore, because of network issues:
+    os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK"] = (
+        "https://api.nuget.org/v3/index.json https://abc.def:8000/packages/"
+    )
+
+    codeql.database.create(build_mode="none")