Add diagnostics rule for arbitrary build messages

Python: Add test of keyword argument with same name as positional-only parameter

config/identical-files.json

@@ -1,6 +1,28 @@
 {
   "DataFlow Java/C++/C#/Go/Python/Ruby/Swift": [
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlow.qll",
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlow.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlow.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlow.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlow.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/DataFlow.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlow.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlow.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlow.qll"
+  ],
+  "DataFlowImpl Java/C++/C#/Go/Python/Ruby/Swift": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"
+  ],
+  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift Legacy Configuration": [
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl1.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl2.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl3.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl4.qll",
@@ -8,38 +30,38 @@
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl6.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForSerializability.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForOnActivityResult.qll",
-    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl1.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl4.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplLocal.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl1.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll",
-    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl1.qll",
     "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll",
     "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl1.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl2.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplForContentDataFlow.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl1.qll",
     "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl2.qll",
     "go/ql/lib/semmle/go/dataflow/internal/DataFlowImplForStringsNewReplacer.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl1.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplForRegExp.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl1.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForHttpClientLibraries.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForPathname.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl1.qll"
   ],
   "DataFlow Java/C++/C#/Go/Python/Ruby/Swift Common": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll",
@@ -52,7 +74,18 @@
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll"
   ],
-  "TaintTracking::Configuration Java/C++/C#/Go/Python/Ruby/Swift": [
+  "TaintTracking Java/C++/C#/Go/Python/Ruby/Swift": [
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTracking.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTracking.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTracking.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking1/TaintTracking.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/tainttracking1/TaintTracking.qll",
+    "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTracking.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking1/TaintTracking.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTracking.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/tainttracking1/TaintTracking.qll"
+  ],
+  "TaintTracking Legacy Configuration Java/C++/C#/Go/Python/Ruby/Swift": [
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -1,5 +1,6 @@
 using Xunit;
 using Semmle.Autobuild.Shared;
+using Semmle.Util;
 using System.Collections.Generic;
 using System;
 using System.Linq;
@@ -75,6 +76,15 @@ namespace Semmle.Autobuild.Cpp.Tests
             throw new ArgumentException("Missing RunProcess " + pattern);
         }
 
+        int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
+        {
+            var ret = (this as IBuildActions).RunProcess(cmd, args, workingDirectory, env, out var stdout);
+
+            stdout.ForEach(line => onOutput(line));
+
+            return ret;
+        }
+
         public IList<string> DirectoryDeleteIn = new List<string>();
 
         void IBuildActions.DirectoryDelete(string dir, bool recursive)
@@ -184,6 +194,15 @@ namespace Semmle.Autobuild.Cpp.Tests
             if (!DownloadFiles.Contains((address, fileName)))
                 throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");
         }
+
+        public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new TestDiagnosticWriter();
+    }
+
+    internal class TestDiagnosticWriter : IDiagnosticsWriter
+    {
+        public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
+
+        public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
     }
 
     /// <summary>
@@ -243,6 +262,7 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
+            Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
             Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";

cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs

@@ -1,4 +1,5 @@
 using Semmle.Autobuild.Shared;
+using Semmle.Util;
 
 namespace Semmle.Autobuild.Cpp
 {
@@ -21,7 +22,7 @@ namespace Semmle.Autobuild.Cpp
 
     public class CppAutobuilder : Autobuilder<CppAutobuildOptions>
     {
-        public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options) { }
+        public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options, new DiagnosticClassifier()) { }
 
         public override BuildScript GetBuildScript()
         {

cpp/ql/lib/change-notes/2023-03-02-dataflow-conf-module.md

@@ -0,0 +1,9 @@
+---
+category: majorAnalysis
+---
+* The main data flow and taint tracking APIs have been changed. The old APIs
+  remain in place for now and translate to the new through a
+  backwards-compatible wrapper. If multiple configurations are in scope
+  simultaneously, then this may affect results slightly. The new API is quite
+  similar to the old, but makes use of a configuration module instead of a
+  configuration class.

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/DataFlow.qll

@@ -22,5 +22,6 @@
 import cpp
 
 module DataFlow {
-  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlow
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl1
 }

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/TaintTracking.qll

@@ -19,5 +19,6 @@ import semmle.code.cpp.ir.dataflow.DataFlow
 import semmle.code.cpp.ir.dataflow.DataFlow2
 
 module TaintTracking {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.tainttracking1.TaintTracking
   import experimental.semmle.code.cpp.ir.dataflow.internal.tainttracking1.TaintTrackingImpl
 }

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlow.qll

@@ -0,0 +1,245 @@
+/**
+ * Provides an implementation of global (interprocedural) data flow. This file
+ * re-exports the local (intraprocedural) data flow analysis from
+ * `DataFlowImplSpecific::Public` and adds a global analysis, mainly exposed
+ * through the `Make` and `MakeWithState` modules.
+ */
+
+private import DataFlowImplCommon
+private import DataFlowImplSpecific::Private
+import DataFlowImplSpecific::Public
+import DataFlowImplCommonPublic
+private import DataFlowImpl
+
+/** An input configuration for data flow. */
+signature module ConfigSig {
+  /**
+   * Holds if `source` is a relevant data flow source.
+   */
+  predicate isSource(Node source);
+
+  /**
+   * Holds if `sink` is a relevant data flow sink.
+   */
+  predicate isSink(Node sink);
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  default predicate isBarrier(Node node) { none() }
+
+  /** Holds if data flow into `node` is prohibited. */
+  default predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  default predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  default predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  default predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  default int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  default FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  default predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  default predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (as it is in a `path-problem` query).
+   */
+  default predicate includeHiddenNodes() { none() }
+}
+
+/** An input configuration for data flow using flow state. */
+signature module StateConfigSig {
+  bindingset[this]
+  class FlowState;
+
+  /**
+   * Holds if `source` is a relevant data flow source with the given initial
+   * `state`.
+   */
+  predicate isSource(Node source, FlowState state);
+
+  /**
+   * Holds if `sink` is a relevant data flow sink accepting `state`.
+   */
+  predicate isSink(Node sink, FlowState state);
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  default predicate isBarrier(Node node) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited when the flow state is
+   * `state`.
+   */
+  predicate isBarrier(Node node, FlowState state);
+
+  /** Holds if data flow into `node` is prohibited. */
+  default predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  default predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  default predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
+   */
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2);
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  default predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  default int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  default FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  default predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  default predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (as it is in a `path-problem` query).
+   */
+  default predicate includeHiddenNodes() { none() }
+}
+
+/**
+ * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
+ * measured in approximate number of interprocedural steps.
+ */
+signature int explorationLimitSig();
+
+/**
+ * The output of a data flow computation.
+ */
+signature module DataFlowSig {
+  /**
+   * A `Node` augmented with a call context (except for sinks) and an access path.
+   * Only those `PathNode`s that are reachable from a source, and which can reach a sink, are generated.
+   */
+  class PathNode;
+
+  /**
+   * Holds if data can flow from `source` to `sink`.
+   *
+   * The corresponding paths are generated from the end-points and the graph
+   * included in the module `PathGraph`.
+   */
+  predicate hasFlowPath(PathNode source, PathNode sink);
+
+  /**
+   * Holds if data can flow from `source` to `sink`.
+   */
+  predicate hasFlow(Node source, Node sink);
+
+  /**
+   * Holds if data can flow from some source to `sink`.
+   */
+  predicate hasFlowTo(Node sink);
+
+  /**
+   * Holds if data can flow from some source to `sink`.
+   */
+  predicate hasFlowToExpr(DataFlowExpr sink);
+}
+
+/**
+ * Constructs a standard data flow computation.
+ */
+module Make<ConfigSig Config> implements DataFlowSig {
+  private module C implements FullStateConfigSig {
+    import DefaultState<Config>
+    import Config
+  }
+
+  import Impl<C>
+}
+
+/**
+ * Constructs a data flow computation using flow state.
+ */
+module MakeWithState<StateConfigSig Config> implements DataFlowSig {
+  private module C implements FullStateConfigSig {
+    import Config
+  }
+
+  import Impl<C>
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl1.qll

@@ -0,0 +1,396 @@
+/**
+ * DEPRECATED: Use `Make` and `MakeWithState` instead.
+ *
+ * Provides a `Configuration` class backwards-compatible interface to the data
+ * flow library.
+ */
+
+private import DataFlowImplCommon
+private import DataFlowImplSpecific::Private
+import DataFlowImplSpecific::Public
+private import DataFlowImpl
+import DataFlowImplCommonPublic
+import FlowStateString
+
+/**
+ * A configuration of interprocedural data flow analysis. This defines
+ * sources, sinks, and any other configurable aspect of the analysis. Each
+ * use of the global data flow library must define its own unique extension
+ * of this abstract class. To create a configuration, extend this class with
+ * a subclass whose characteristic predicate is a unique singleton string.
+ * For example, write
+ *
+ * ```ql
+ * class MyAnalysisConfiguration extends DataFlow::Configuration {
+ *   MyAnalysisConfiguration() { this = "MyAnalysisConfiguration" }
+ *   // Override `isSource` and `isSink`.
+ *   // Optionally override `isBarrier`.
+ *   // Optionally override `isAdditionalFlowStep`.
+ * }
+ * ```
+ * Conceptually, this defines a graph where the nodes are `DataFlow::Node`s and
+ * the edges are those data-flow steps that preserve the value of the node
+ * along with any additional edges defined by `isAdditionalFlowStep`.
+ * Specifying nodes in `isBarrier` will remove those nodes from the graph, and
+ * specifying nodes in `isBarrierIn` and/or `isBarrierOut` will remove in-going
+ * and/or out-going edges from those nodes, respectively.
+ *
+ * Then, to query whether there is flow between some `source` and `sink`,
+ * write
+ *
+ * ```ql
+ * exists(MyAnalysisConfiguration cfg | cfg.hasFlow(source, sink))
+ * ```
+ *
+ * Multiple configurations can coexist, but two classes extending
+ * `DataFlow::Configuration` should never depend on each other. One of them
+ * should instead depend on a `DataFlow2::Configuration`, a
+ * `DataFlow3::Configuration`, or a `DataFlow4::Configuration`.
+ */
+abstract class Configuration extends string {
+  bindingset[this]
+  Configuration() { any() }
+
+  /**
+   * Holds if `source` is a relevant data flow source.
+   */
+  predicate isSource(Node source) { none() }
+
+  /**
+   * Holds if `source` is a relevant data flow source with the given initial
+   * `state`.
+   */
+  predicate isSource(Node source, FlowState state) { none() }
+
+  /**
+   * Holds if `sink` is a relevant data flow sink.
+   */
+  predicate isSink(Node sink) { none() }
+
+  /**
+   * Holds if `sink` is a relevant data flow sink accepting `state`.
+   */
+  predicate isSink(Node sink, FlowState state) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  predicate isBarrier(Node node) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited when the flow state is
+   * `state`.
+   */
+  predicate isBarrier(Node node, FlowState state) { none() }
+
+  /** Holds if data flow into `node` is prohibited. */
+  predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * DEPRECATED: Use `isBarrier` and `BarrierGuard` module instead.
+   *
+   * Holds if data flow through nodes guarded by `guard` is prohibited.
+   */
+  deprecated predicate isBarrierGuard(BarrierGuard guard) { none() }
+
+  /**
+   * DEPRECATED: Use `isBarrier` and `BarrierGuard` module instead.
+   *
+   * Holds if data flow through nodes guarded by `guard` is prohibited when
+   * the flow state is `state`
+   */
+  deprecated predicate isBarrierGuard(BarrierGuard guard, FlowState state) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
+   */
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
+    none()
+  }
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if data may flow from `source` to `sink` for this configuration.
+   */
+  predicate hasFlow(Node source, Node sink) { hasFlow(source, sink, this) }
+
+  /**
+   * Holds if data may flow from `source` to `sink` for this configuration.
+   *
+   * The corresponding paths are generated from the end-points and the graph
+   * included in the module `PathGraph`.
+   */
+  predicate hasFlowPath(PathNode source, PathNode sink) { hasFlowPath(source, sink, this) }
+
+  /**
+   * Holds if data may flow from some source to `sink` for this configuration.
+   */
+  predicate hasFlowTo(Node sink) { hasFlowTo(sink, this) }
+
+  /**
+   * Holds if data may flow from some source to `sink` for this configuration.
+   */
+  predicate hasFlowToExpr(DataFlowExpr sink) { this.hasFlowTo(exprNode(sink)) }
+
+  /**
+   * DEPRECATED: Use `FlowExploration<explorationLimit>` instead.
+   *
+   * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
+   * measured in approximate number of interprocedural steps.
+   */
+  deprecated int explorationLimit() { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+}
+
+/**
+ * This class exists to prevent mutual recursion between the user-overridden
+ * member predicates of `Configuration` and the rest of the data-flow library.
+ * Good performance cannot be guaranteed in the presence of such recursion, so
+ * it should be replaced by using more than one copy of the data flow library.
+ */
+abstract private class ConfigurationRecursionPrevention extends Configuration {
+  bindingset[this]
+  ConfigurationRecursionPrevention() { any() }
+
+  override predicate hasFlow(Node source, Node sink) {
+    strictcount(Node n | this.isSource(n)) < 0
+    or
+    strictcount(Node n | this.isSource(n, _)) < 0
+    or
+    strictcount(Node n | this.isSink(n)) < 0
+    or
+    strictcount(Node n | this.isSink(n, _)) < 0
+    or
+    strictcount(Node n1, Node n2 | this.isAdditionalFlowStep(n1, n2)) < 0
+    or
+    strictcount(Node n1, Node n2 | this.isAdditionalFlowStep(n1, _, n2, _)) < 0
+    or
+    super.hasFlow(source, sink)
+  }
+}
+
+/** A bridge class to access the deprecated `isBarrierGuard`. */
+private class BarrierGuardGuardedNodeBridge extends Unit {
+  abstract predicate guardedNode(Node n, Configuration config);
+
+  abstract predicate guardedNode(Node n, FlowState state, Configuration config);
+}
+
+private class BarrierGuardGuardedNode extends BarrierGuardGuardedNodeBridge {
+  deprecated override predicate guardedNode(Node n, Configuration config) {
+    exists(BarrierGuard g |
+      config.isBarrierGuard(g) and
+      n = g.getAGuardedNode()
+    )
+  }
+
+  deprecated override predicate guardedNode(Node n, FlowState state, Configuration config) {
+    exists(BarrierGuard g |
+      config.isBarrierGuard(g, state) and
+      n = g.getAGuardedNode()
+    )
+  }
+}
+
+private FlowState relevantState(Configuration config) {
+  config.isSource(_, result) or
+  config.isSink(_, result) or
+  config.isBarrier(_, result) or
+  config.isAdditionalFlowStep(_, result, _, _) or
+  config.isAdditionalFlowStep(_, _, _, result)
+}
+
+private newtype TConfigState =
+  TMkConfigState(Configuration config, FlowState state) {
+    state = relevantState(config) or state instanceof FlowStateEmpty
+  }
+
+private Configuration getConfig(TConfigState state) { state = TMkConfigState(result, _) }
+
+private FlowState getState(TConfigState state) { state = TMkConfigState(_, result) }
+
+private predicate singleConfiguration() { 1 = strictcount(Configuration c) }
+
+private module Config implements FullStateConfigSig {
+  class FlowState = TConfigState;
+
+  predicate isSource(Node source, FlowState state) {
+    getConfig(state).isSource(source, getState(state))
+    or
+    getConfig(state).isSource(source) and getState(state) instanceof FlowStateEmpty
+  }
+
+  predicate isSink(Node sink, FlowState state) {
+    getConfig(state).isSink(sink, getState(state))
+    or
+    getConfig(state).isSink(sink) and getState(state) instanceof FlowStateEmpty
+  }
+
+  predicate isBarrier(Node node) { none() }
+
+  predicate isBarrier(Node node, FlowState state) {
+    getConfig(state).isBarrier(node, getState(state)) or
+    getConfig(state).isBarrier(node) or
+    any(BarrierGuardGuardedNodeBridge b).guardedNode(node, getState(state), getConfig(state)) or
+    any(BarrierGuardGuardedNodeBridge b).guardedNode(node, getConfig(state))
+  }
+
+  predicate isBarrierIn(Node node) { any(Configuration config).isBarrierIn(node) }
+
+  predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
+
+  predicate isAdditionalFlowStep(Node node1, Node node2) {
+    singleConfiguration() and
+    any(Configuration config).isAdditionalFlowStep(node1, node2)
+  }
+
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
+    getConfig(state1).isAdditionalFlowStep(node1, getState(state1), node2, getState(state2)) and
+    getConfig(state2) = getConfig(state1)
+    or
+    not singleConfiguration() and
+    getConfig(state1).isAdditionalFlowStep(node1, node2) and
+    state2 = state1
+  }
+
+  predicate allowImplicitRead(Node node, ContentSet c) {
+    any(Configuration config).allowImplicitRead(node, c)
+  }
+
+  int fieldFlowBranchLimit() { result = min(any(Configuration config).fieldFlowBranchLimit()) }
+
+  FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
+
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
+  predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
+}
+
+private import Impl<Config> as I
+import I
+
+/**
+ * A `Node` augmented with a call context (except for sinks), an access path, and a configuration.
+ * Only those `PathNode`s that are reachable from a source, and which can reach a sink, are generated.
+ */
+class PathNode instanceof I::PathNode {
+  /** Gets a textual representation of this element. */
+  final string toString() { result = super.toString() }
+
+  /**
+   * Gets a textual representation of this element, including a textual
+   * representation of the call context.
+   */
+  final string toStringWithContext() { result = super.toStringWithContext() }
+
+  /**
+   * Holds if this element is at the specified location.
+   * The location spans column `startcolumn` of line `startline` to
+   * column `endcolumn` of line `endline` in file `filepath`.
+   * For more information, see
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
+   */
+  final predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    super.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+  }
+
+  /** Gets the underlying `Node`. */
+  final Node getNode() { result = super.getNode() }
+
+  /** Gets the `FlowState` of this node. */
+  final FlowState getState() { result = getState(super.getState()) }
+
+  /** Gets the associated configuration. */
+  final Configuration getConfiguration() { result = getConfig(super.getState()) }
+
+  /** Gets a successor of this node, if any. */
+  final PathNode getASuccessor() { result = super.getASuccessor() }
+
+  /** Holds if this node is a source. */
+  final predicate isSource() { super.isSource() }
+
+  /** Holds if this node is a grouping of source nodes. */
+  final predicate isSourceGroup(string group) { super.isSourceGroup(group) }
+
+  /** Holds if this node is a grouping of sink nodes. */
+  final predicate isSinkGroup(string group) { super.isSinkGroup(group) }
+}
+
+private predicate hasFlow(Node source, Node sink, Configuration config) {
+  exists(PathNode source0, PathNode sink0 |
+    hasFlowPath(source0, sink0, config) and
+    source0.getNode() = source and
+    sink0.getNode() = sink
+  )
+}
+
+private predicate hasFlowPath(PathNode source, PathNode sink, Configuration config) {
+  hasFlowPath(source, sink) and source.getConfiguration() = config
+}
+
+private predicate hasFlowTo(Node sink, Configuration config) { hasFlow(_, sink, config) }
+
+predicate flowsTo = hasFlow/3;

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll

@@ -3,6 +3,8 @@ private import DataFlowImplSpecific::Public
 import Cached
 
 module DataFlowImplCommonPublic {
+  /** Provides `FlowState = string`. */
+  module FlowStateString {
     /** A state value to track during data flow. */
     class FlowState = string;
 
@@ -13,6 +15,7 @@ module DataFlowImplCommonPublic {
     class FlowStateEmpty extends FlowState {
       FlowStateEmpty() { this = "" }
     }
+  }
 
   private newtype TFlowFeature =
     TFeatureHasSourceCallContext() or

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll

@@ -565,3 +565,12 @@ private class MyConsistencyConfiguration extends Consistency::ConsistencyConfigu
     any()
   }
 }
+
+/**
+ * Gets an additional term that is added to the `join` and `branch` computations to reflect
+ * an additional forward or backwards branching factor that is not taken into account
+ * when calculating the (virtual) dispatch cost.
+ *
+ * Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter.
+ */
+int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p) { none() }

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/PrintIRLocalFlow.qll

@@ -97,23 +97,23 @@ private string getNodeProperty(DataFlow::Node node, string key) {
     |
       kind, ", "
     )
-  or
+  // or
-  // Is there partial flow from a source to this node?
+  // // Is there partial flow from a source to this node?
-  // This property will only be emitted if partial flow is enabled by overriding
+  // // This property will only be emitted if partial flow is enabled by overriding
-  // `DataFlow::Configuration::explorationLimit()`.
+  // // `DataFlow::Configuration::explorationLimit()`.
-  key = "pflow" and
+  // key = "pflow" and
-  result =
+  // result =
-    strictconcat(DataFlow::PartialPathNode sourceNode, DataFlow::PartialPathNode destNode, int dist,
+  //   strictconcat(DataFlow::PartialPathNode sourceNode, DataFlow::PartialPathNode destNode, int dist,
-      int order1, int order2 |
+  //     int order1, int order2 |
-      any(DataFlow::Configuration cfg).hasPartialFlow(sourceNode, destNode, dist) and
+  //     any(DataFlow::Configuration cfg).hasPartialFlow(sourceNode, destNode, dist) and
-      destNode.getNode() = node and
+  //     destNode.getNode() = node and
-      // Only print flow from a source in the same function.
+  //     // Only print flow from a source in the same function.
-      sourceNode.getNode().getEnclosingCallable() = node.getEnclosingCallable()
+  //     sourceNode.getNode().getEnclosingCallable() = node.getEnclosingCallable()
-    |
+  //   |
-      nodeId(sourceNode.getNode(), order1, order2) + "+" + dist.toString(), ", "
+  //     nodeId(sourceNode.getNode(), order1, order2) + "+" + dist.toString(), ", "
-      order by
+  //     order by
-        order1, order2, dist desc
+  //       order1, order2, dist desc
-    )
+  //   )
 }
 
 /**

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTracking.qll

@@ -0,0 +1,63 @@
+/**
+ * Provides classes for performing local (intra-procedural) and
+ * global (inter-procedural) taint-tracking analyses.
+ */
+
+import TaintTrackingParameter::Public
+private import TaintTrackingParameter::Private
+
+private module AddTaintDefaults<DataFlowInternal::FullStateConfigSig Config> implements
+DataFlowInternal::FullStateConfigSig {
+  import Config
+
+  predicate isBarrier(DataFlow::Node node) {
+    Config::isBarrier(node) or defaultTaintSanitizer(node)
+  }
+
+  predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
+    Config::isAdditionalFlowStep(node1, node2) or
+    defaultAdditionalTaintStep(node1, node2)
+  }
+
+  predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c) {
+    Config::allowImplicitRead(node, c)
+    or
+    (
+      Config::isSink(node, _) or
+      Config::isAdditionalFlowStep(node, _) or
+      Config::isAdditionalFlowStep(node, _, _, _)
+    ) and
+    defaultImplicitTaintRead(node, c)
+  }
+}
+
+/**
+ * Constructs a standard taint tracking computation.
+ */
+module Make<DataFlow::ConfigSig Config> implements DataFlow::DataFlowSig {
+  private module Config0 implements DataFlowInternal::FullStateConfigSig {
+    import DataFlowInternal::DefaultState<Config>
+    import Config
+  }
+
+  private module C implements DataFlowInternal::FullStateConfigSig {
+    import AddTaintDefaults<Config0>
+  }
+
+  import DataFlowInternal::Impl<C>
+}
+
+/**
+ * Constructs a taint tracking computation using flow state.
+ */
+module MakeWithState<DataFlow::StateConfigSig Config> implements DataFlow::DataFlowSig {
+  private module Config0 implements DataFlowInternal::FullStateConfigSig {
+    import Config
+  }
+
+  private module C implements DataFlowInternal::FullStateConfigSig {
+    import AddTaintDefaults<Config0>
+  }
+
+  import DataFlowInternal::Impl<C>
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingParameter.qll

@@ -2,4 +2,5 @@ import experimental.semmle.code.cpp.ir.dataflow.internal.TaintTrackingUtil as Pu
 
 module Private {
   import experimental.semmle.code.cpp.ir.dataflow.DataFlow::DataFlow as DataFlow
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl as DataFlowInternal
 }

cpp/ql/lib/semmle/code/cpp/Declaration.qll

@@ -68,7 +68,9 @@ class Declaration extends Locatable, @declaration {
    * Holds if this declaration has the fully-qualified name `qualifiedName`.
    * See `getQualifiedName`.
    */
-  predicate hasQualifiedName(string qualifiedName) { this.getQualifiedName() = qualifiedName }
+  deprecated predicate hasQualifiedName(string qualifiedName) {
+    this.getQualifiedName() = qualifiedName
+  }
 
   /**
    * Holds if this declaration has a fully-qualified name with a name-space
@@ -184,7 +186,7 @@ class Declaration extends Locatable, @declaration {
   predicate hasDefinition() { exists(this.getDefinition()) }
 
   /** DEPRECATED: Use `hasDefinition` instead. */
-  predicate isDefined() { this.hasDefinition() }
+  deprecated predicate isDefined() { this.hasDefinition() }
 
   /** Gets the preferred location of this declaration, if any. */
   override Location getLocation() { none() }

cpp/ql/lib/semmle/code/cpp/Function.qll

@@ -41,7 +41,7 @@ class Function extends Declaration, ControlFlowNode, AccessHolder, @function {
    * `min<int>(int, int) -> int`, and the full signature of the uninstantiated
    * template on the first line would be `min<T>(T, T) -> T`.
    */
-  string getFullSignature() {
+  deprecated string getFullSignature() {
     exists(string name, string templateArgs, string args |
       result = name + templateArgs + args + " -> " + this.getType().toString() and
       name = this.getQualifiedName() and

cpp/ql/lib/semmle/code/cpp/commons/Alloc.qll

@@ -12,7 +12,9 @@ predicate freeFunction(Function f, int argNum) { argNum = f.(DeallocationFunctio
  *
  * DEPRECATED: Use `DeallocationExpr` instead (this also includes `delete` expressions).
  */
-predicate freeCall(FunctionCall fc, Expr arg) { arg = fc.(DeallocationExpr).getFreedExpr() }
+deprecated predicate freeCall(FunctionCall fc, Expr arg) {
+  arg = fc.(DeallocationExpr).getFreedExpr()
+}
 
 /**
  * Is e some kind of allocation or deallocation (`new`, `alloc`, `realloc`, `delete`, `free` etc)?

cpp/ql/lib/semmle/code/cpp/dataflow/DataFlow.qll

@@ -20,5 +20,6 @@
 import cpp
 
 module DataFlow {
-  import semmle.code.cpp.dataflow.internal.DataFlowImpl
+  import semmle.code.cpp.dataflow.internal.DataFlow
+  import semmle.code.cpp.dataflow.internal.DataFlowImpl1
 }

cpp/ql/lib/semmle/code/cpp/dataflow/TaintTracking.qll

@@ -19,5 +19,6 @@ import semmle.code.cpp.dataflow.DataFlow
 import semmle.code.cpp.dataflow.DataFlow2
 
 module TaintTracking {
+  import semmle.code.cpp.dataflow.internal.tainttracking1.TaintTracking
   import semmle.code.cpp.dataflow.internal.tainttracking1.TaintTrackingImpl
 }

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlow.qll

@@ -0,0 +1,245 @@
+/**
+ * Provides an implementation of global (interprocedural) data flow. This file
+ * re-exports the local (intraprocedural) data flow analysis from
+ * `DataFlowImplSpecific::Public` and adds a global analysis, mainly exposed
+ * through the `Make` and `MakeWithState` modules.
+ */
+
+private import DataFlowImplCommon
+private import DataFlowImplSpecific::Private
+import DataFlowImplSpecific::Public
+import DataFlowImplCommonPublic
+private import DataFlowImpl
+
+/** An input configuration for data flow. */
+signature module ConfigSig {
+  /**
+   * Holds if `source` is a relevant data flow source.
+   */
+  predicate isSource(Node source);
+
+  /**
+   * Holds if `sink` is a relevant data flow sink.
+   */
+  predicate isSink(Node sink);
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  default predicate isBarrier(Node node) { none() }
+
+  /** Holds if data flow into `node` is prohibited. */
+  default predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  default predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  default predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  default predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  default int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  default FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  default predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  default predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (as it is in a `path-problem` query).
+   */
+  default predicate includeHiddenNodes() { none() }
+}
+
+/** An input configuration for data flow using flow state. */
+signature module StateConfigSig {
+  bindingset[this]
+  class FlowState;
+
+  /**
+   * Holds if `source` is a relevant data flow source with the given initial
+   * `state`.
+   */
+  predicate isSource(Node source, FlowState state);
+
+  /**
+   * Holds if `sink` is a relevant data flow sink accepting `state`.
+   */
+  predicate isSink(Node sink, FlowState state);
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  default predicate isBarrier(Node node) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited when the flow state is
+   * `state`.
+   */
+  predicate isBarrier(Node node, FlowState state);
+
+  /** Holds if data flow into `node` is prohibited. */
+  default predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  default predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  default predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
+   */
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2);
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  default predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  default int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  default FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  default predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  default predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (as it is in a `path-problem` query).
+   */
+  default predicate includeHiddenNodes() { none() }
+}
+
+/**
+ * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
+ * measured in approximate number of interprocedural steps.
+ */
+signature int explorationLimitSig();
+
+/**
+ * The output of a data flow computation.
+ */
+signature module DataFlowSig {
+  /**
+   * A `Node` augmented with a call context (except for sinks) and an access path.
+   * Only those `PathNode`s that are reachable from a source, and which can reach a sink, are generated.
+   */
+  class PathNode;
+
+  /**
+   * Holds if data can flow from `source` to `sink`.
+   *
+   * The corresponding paths are generated from the end-points and the graph
+   * included in the module `PathGraph`.
+   */
+  predicate hasFlowPath(PathNode source, PathNode sink);
+
+  /**
+   * Holds if data can flow from `source` to `sink`.
+   */
+  predicate hasFlow(Node source, Node sink);
+
+  /**
+   * Holds if data can flow from some source to `sink`.
+   */
+  predicate hasFlowTo(Node sink);
+
+  /**
+   * Holds if data can flow from some source to `sink`.
+   */
+  predicate hasFlowToExpr(DataFlowExpr sink);
+}
+
+/**
+ * Constructs a standard data flow computation.
+ */
+module Make<ConfigSig Config> implements DataFlowSig {
+  private module C implements FullStateConfigSig {
+    import DefaultState<Config>
+    import Config
+  }
+
+  import Impl<C>
+}
+
+/**
+ * Constructs a data flow computation using flow state.
+ */
+module MakeWithState<StateConfigSig Config> implements DataFlowSig {
+  private module C implements FullStateConfigSig {
+    import Config
+  }
+
+  import Impl<C>
+}

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl1.qll

@@ -0,0 +1,396 @@
+/**
+ * DEPRECATED: Use `Make` and `MakeWithState` instead.
+ *
+ * Provides a `Configuration` class backwards-compatible interface to the data
+ * flow library.
+ */
+
+private import DataFlowImplCommon
+private import DataFlowImplSpecific::Private
+import DataFlowImplSpecific::Public
+private import DataFlowImpl
+import DataFlowImplCommonPublic
+import FlowStateString
+
+/**
+ * A configuration of interprocedural data flow analysis. This defines
+ * sources, sinks, and any other configurable aspect of the analysis. Each
+ * use of the global data flow library must define its own unique extension
+ * of this abstract class. To create a configuration, extend this class with
+ * a subclass whose characteristic predicate is a unique singleton string.
+ * For example, write
+ *
+ * ```ql
+ * class MyAnalysisConfiguration extends DataFlow::Configuration {
+ *   MyAnalysisConfiguration() { this = "MyAnalysisConfiguration" }
+ *   // Override `isSource` and `isSink`.
+ *   // Optionally override `isBarrier`.
+ *   // Optionally override `isAdditionalFlowStep`.
+ * }
+ * ```
+ * Conceptually, this defines a graph where the nodes are `DataFlow::Node`s and
+ * the edges are those data-flow steps that preserve the value of the node
+ * along with any additional edges defined by `isAdditionalFlowStep`.
+ * Specifying nodes in `isBarrier` will remove those nodes from the graph, and
+ * specifying nodes in `isBarrierIn` and/or `isBarrierOut` will remove in-going
+ * and/or out-going edges from those nodes, respectively.
+ *
+ * Then, to query whether there is flow between some `source` and `sink`,
+ * write
+ *
+ * ```ql
+ * exists(MyAnalysisConfiguration cfg | cfg.hasFlow(source, sink))
+ * ```
+ *
+ * Multiple configurations can coexist, but two classes extending
+ * `DataFlow::Configuration` should never depend on each other. One of them
+ * should instead depend on a `DataFlow2::Configuration`, a
+ * `DataFlow3::Configuration`, or a `DataFlow4::Configuration`.
+ */
+abstract class Configuration extends string {
+  bindingset[this]
+  Configuration() { any() }
+
+  /**
+   * Holds if `source` is a relevant data flow source.
+   */
+  predicate isSource(Node source) { none() }
+
+  /**
+   * Holds if `source` is a relevant data flow source with the given initial
+   * `state`.
+   */
+  predicate isSource(Node source, FlowState state) { none() }
+
+  /**
+   * Holds if `sink` is a relevant data flow sink.
+   */
+  predicate isSink(Node sink) { none() }
+
+  /**
+   * Holds if `sink` is a relevant data flow sink accepting `state`.
+   */
+  predicate isSink(Node sink, FlowState state) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  predicate isBarrier(Node node) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited when the flow state is
+   * `state`.
+   */
+  predicate isBarrier(Node node, FlowState state) { none() }
+
+  /** Holds if data flow into `node` is prohibited. */
+  predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * DEPRECATED: Use `isBarrier` and `BarrierGuard` module instead.
+   *
+   * Holds if data flow through nodes guarded by `guard` is prohibited.
+   */
+  deprecated predicate isBarrierGuard(BarrierGuard guard) { none() }
+
+  /**
+   * DEPRECATED: Use `isBarrier` and `BarrierGuard` module instead.
+   *
+   * Holds if data flow through nodes guarded by `guard` is prohibited when
+   * the flow state is `state`
+   */
+  deprecated predicate isBarrierGuard(BarrierGuard guard, FlowState state) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
+   */
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
+    none()
+  }
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if data may flow from `source` to `sink` for this configuration.
+   */
+  predicate hasFlow(Node source, Node sink) { hasFlow(source, sink, this) }
+
+  /**
+   * Holds if data may flow from `source` to `sink` for this configuration.
+   *
+   * The corresponding paths are generated from the end-points and the graph
+   * included in the module `PathGraph`.
+   */
+  predicate hasFlowPath(PathNode source, PathNode sink) { hasFlowPath(source, sink, this) }
+
+  /**
+   * Holds if data may flow from some source to `sink` for this configuration.
+   */
+  predicate hasFlowTo(Node sink) { hasFlowTo(sink, this) }
+
+  /**
+   * Holds if data may flow from some source to `sink` for this configuration.
+   */
+  predicate hasFlowToExpr(DataFlowExpr sink) { this.hasFlowTo(exprNode(sink)) }
+
+  /**
+   * DEPRECATED: Use `FlowExploration<explorationLimit>` instead.
+   *
+   * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
+   * measured in approximate number of interprocedural steps.
+   */
+  deprecated int explorationLimit() { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+}
+
+/**
+ * This class exists to prevent mutual recursion between the user-overridden
+ * member predicates of `Configuration` and the rest of the data-flow library.
+ * Good performance cannot be guaranteed in the presence of such recursion, so
+ * it should be replaced by using more than one copy of the data flow library.
+ */
+abstract private class ConfigurationRecursionPrevention extends Configuration {
+  bindingset[this]
+  ConfigurationRecursionPrevention() { any() }
+
+  override predicate hasFlow(Node source, Node sink) {
+    strictcount(Node n | this.isSource(n)) < 0
+    or
+    strictcount(Node n | this.isSource(n, _)) < 0
+    or
+    strictcount(Node n | this.isSink(n)) < 0
+    or
+    strictcount(Node n | this.isSink(n, _)) < 0
+    or
+    strictcount(Node n1, Node n2 | this.isAdditionalFlowStep(n1, n2)) < 0
+    or
+    strictcount(Node n1, Node n2 | this.isAdditionalFlowStep(n1, _, n2, _)) < 0
+    or
+    super.hasFlow(source, sink)
+  }
+}
+
+/** A bridge class to access the deprecated `isBarrierGuard`. */
+private class BarrierGuardGuardedNodeBridge extends Unit {
+  abstract predicate guardedNode(Node n, Configuration config);
+
+  abstract predicate guardedNode(Node n, FlowState state, Configuration config);
+}
+
+private class BarrierGuardGuardedNode extends BarrierGuardGuardedNodeBridge {
+  deprecated override predicate guardedNode(Node n, Configuration config) {
+    exists(BarrierGuard g |
+      config.isBarrierGuard(g) and
+      n = g.getAGuardedNode()
+    )
+  }
+
+  deprecated override predicate guardedNode(Node n, FlowState state, Configuration config) {
+    exists(BarrierGuard g |
+      config.isBarrierGuard(g, state) and
+      n = g.getAGuardedNode()
+    )
+  }
+}
+
+private FlowState relevantState(Configuration config) {
+  config.isSource(_, result) or
+  config.isSink(_, result) or
+  config.isBarrier(_, result) or
+  config.isAdditionalFlowStep(_, result, _, _) or
+  config.isAdditionalFlowStep(_, _, _, result)
+}
+
+private newtype TConfigState =
+  TMkConfigState(Configuration config, FlowState state) {
+    state = relevantState(config) or state instanceof FlowStateEmpty
+  }
+
+private Configuration getConfig(TConfigState state) { state = TMkConfigState(result, _) }
+
+private FlowState getState(TConfigState state) { state = TMkConfigState(_, result) }
+
+private predicate singleConfiguration() { 1 = strictcount(Configuration c) }
+
+private module Config implements FullStateConfigSig {
+  class FlowState = TConfigState;
+
+  predicate isSource(Node source, FlowState state) {
+    getConfig(state).isSource(source, getState(state))
+    or
+    getConfig(state).isSource(source) and getState(state) instanceof FlowStateEmpty
+  }
+
+  predicate isSink(Node sink, FlowState state) {
+    getConfig(state).isSink(sink, getState(state))
+    or
+    getConfig(state).isSink(sink) and getState(state) instanceof FlowStateEmpty
+  }
+
+  predicate isBarrier(Node node) { none() }
+
+  predicate isBarrier(Node node, FlowState state) {
+    getConfig(state).isBarrier(node, getState(state)) or
+    getConfig(state).isBarrier(node) or
+    any(BarrierGuardGuardedNodeBridge b).guardedNode(node, getState(state), getConfig(state)) or
+    any(BarrierGuardGuardedNodeBridge b).guardedNode(node, getConfig(state))
+  }
+
+  predicate isBarrierIn(Node node) { any(Configuration config).isBarrierIn(node) }
+
+  predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
+
+  predicate isAdditionalFlowStep(Node node1, Node node2) {
+    singleConfiguration() and
+    any(Configuration config).isAdditionalFlowStep(node1, node2)
+  }
+
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
+    getConfig(state1).isAdditionalFlowStep(node1, getState(state1), node2, getState(state2)) and
+    getConfig(state2) = getConfig(state1)
+    or
+    not singleConfiguration() and
+    getConfig(state1).isAdditionalFlowStep(node1, node2) and
+    state2 = state1
+  }
+
+  predicate allowImplicitRead(Node node, ContentSet c) {
+    any(Configuration config).allowImplicitRead(node, c)
+  }
+
+  int fieldFlowBranchLimit() { result = min(any(Configuration config).fieldFlowBranchLimit()) }
+
+  FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
+
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
+  predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
+}
+
+private import Impl<Config> as I
+import I
+
+/**
+ * A `Node` augmented with a call context (except for sinks), an access path, and a configuration.
+ * Only those `PathNode`s that are reachable from a source, and which can reach a sink, are generated.
+ */
+class PathNode instanceof I::PathNode {
+  /** Gets a textual representation of this element. */
+  final string toString() { result = super.toString() }
+
+  /**
+   * Gets a textual representation of this element, including a textual
+   * representation of the call context.
+   */
+  final string toStringWithContext() { result = super.toStringWithContext() }
+
+  /**
+   * Holds if this element is at the specified location.
+   * The location spans column `startcolumn` of line `startline` to
+   * column `endcolumn` of line `endline` in file `filepath`.
+   * For more information, see
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
+   */
+  final predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    super.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+  }
+
+  /** Gets the underlying `Node`. */
+  final Node getNode() { result = super.getNode() }
+
+  /** Gets the `FlowState` of this node. */
+  final FlowState getState() { result = getState(super.getState()) }
+
+  /** Gets the associated configuration. */
+  final Configuration getConfiguration() { result = getConfig(super.getState()) }
+
+  /** Gets a successor of this node, if any. */
+  final PathNode getASuccessor() { result = super.getASuccessor() }
+
+  /** Holds if this node is a source. */
+  final predicate isSource() { super.isSource() }
+
+  /** Holds if this node is a grouping of source nodes. */
+  final predicate isSourceGroup(string group) { super.isSourceGroup(group) }
+
+  /** Holds if this node is a grouping of sink nodes. */
+  final predicate isSinkGroup(string group) { super.isSinkGroup(group) }
+}
+
+private predicate hasFlow(Node source, Node sink, Configuration config) {
+  exists(PathNode source0, PathNode sink0 |
+    hasFlowPath(source0, sink0, config) and
+    source0.getNode() = source and
+    sink0.getNode() = sink
+  )
+}
+
+private predicate hasFlowPath(PathNode source, PathNode sink, Configuration config) {
+  hasFlowPath(source, sink) and source.getConfiguration() = config
+}
+
+private predicate hasFlowTo(Node sink, Configuration config) { hasFlow(_, sink, config) }
+
+predicate flowsTo = hasFlow/3;

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll

@@ -3,6 +3,8 @@ private import DataFlowImplSpecific::Public
 import Cached
 
 module DataFlowImplCommonPublic {
+  /** Provides `FlowState = string`. */
+  module FlowStateString {
     /** A state value to track during data flow. */
     class FlowState = string;
 
@@ -13,6 +15,7 @@ module DataFlowImplCommonPublic {
     class FlowStateEmpty extends FlowState {
       FlowStateEmpty() { this = "" }
     }
+  }
 
   private newtype TFlowFeature =
     TFeatureHasSourceCallContext() or

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowPrivate.qll

@@ -318,3 +318,12 @@ private class MyConsistencyConfiguration extends Consistency::ConsistencyConfigu
     // consistency alerts enough that most of them are interesting.
   }
 }
+
+/**
+ * Gets an additional term that is added to the `join` and `branch` computations to reflect
+ * an additional forward or backwards branching factor that is not taken into account
+ * when calculating the (virtual) dispatch cost.
+ *
+ * Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter.
+ */
+int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p) { none() }

cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTracking.qll

@@ -0,0 +1,63 @@
+/**
+ * Provides classes for performing local (intra-procedural) and
+ * global (inter-procedural) taint-tracking analyses.
+ */
+
+import TaintTrackingParameter::Public
+private import TaintTrackingParameter::Private
+
+private module AddTaintDefaults<DataFlowInternal::FullStateConfigSig Config> implements
+DataFlowInternal::FullStateConfigSig {
+  import Config
+
+  predicate isBarrier(DataFlow::Node node) {
+    Config::isBarrier(node) or defaultTaintSanitizer(node)
+  }
+
+  predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
+    Config::isAdditionalFlowStep(node1, node2) or
+    defaultAdditionalTaintStep(node1, node2)
+  }
+
+  predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c) {
+    Config::allowImplicitRead(node, c)
+    or
+    (
+      Config::isSink(node, _) or
+      Config::isAdditionalFlowStep(node, _) or
+      Config::isAdditionalFlowStep(node, _, _, _)
+    ) and
+    defaultImplicitTaintRead(node, c)
+  }
+}
+
+/**
+ * Constructs a standard taint tracking computation.
+ */
+module Make<DataFlow::ConfigSig Config> implements DataFlow::DataFlowSig {
+  private module Config0 implements DataFlowInternal::FullStateConfigSig {
+    import DataFlowInternal::DefaultState<Config>
+    import Config
+  }
+
+  private module C implements DataFlowInternal::FullStateConfigSig {
+    import AddTaintDefaults<Config0>
+  }
+
+  import DataFlowInternal::Impl<C>
+}
+
+/**
+ * Constructs a taint tracking computation using flow state.
+ */
+module MakeWithState<DataFlow::StateConfigSig Config> implements DataFlow::DataFlowSig {
+  private module Config0 implements DataFlowInternal::FullStateConfigSig {
+    import Config
+  }
+
+  private module C implements DataFlowInternal::FullStateConfigSig {
+    import AddTaintDefaults<Config0>
+  }
+
+  import DataFlowInternal::Impl<C>
+}

cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingParameter.qll

@@ -2,4 +2,5 @@ import semmle.code.cpp.dataflow.internal.TaintTrackingUtil as Public
 
 module Private {
   import semmle.code.cpp.dataflow.DataFlow::DataFlow as DataFlow
+  import semmle.code.cpp.dataflow.internal.DataFlowImpl as DataFlowInternal
 }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/DataFlow.qll

@@ -22,5 +22,6 @@
 import cpp
 
 module DataFlow {
-  import semmle.code.cpp.ir.dataflow.internal.DataFlowImpl
+  import semmle.code.cpp.ir.dataflow.internal.DataFlow
+  import semmle.code.cpp.ir.dataflow.internal.DataFlowImpl1
 }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/TaintTracking.qll

@@ -19,5 +19,6 @@ import semmle.code.cpp.ir.dataflow.DataFlow
 import semmle.code.cpp.ir.dataflow.DataFlow2
 
 module TaintTracking {
+  import semmle.code.cpp.ir.dataflow.internal.tainttracking1.TaintTracking
   import semmle.code.cpp.ir.dataflow.internal.tainttracking1.TaintTrackingImpl
 }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlow.qll

@@ -0,0 +1,245 @@
+/**
+ * Provides an implementation of global (interprocedural) data flow. This file
+ * re-exports the local (intraprocedural) data flow analysis from
+ * `DataFlowImplSpecific::Public` and adds a global analysis, mainly exposed
+ * through the `Make` and `MakeWithState` modules.
+ */
+
+private import DataFlowImplCommon
+private import DataFlowImplSpecific::Private
+import DataFlowImplSpecific::Public
+import DataFlowImplCommonPublic
+private import DataFlowImpl
+
+/** An input configuration for data flow. */
+signature module ConfigSig {
+  /**
+   * Holds if `source` is a relevant data flow source.
+   */
+  predicate isSource(Node source);
+
+  /**
+   * Holds if `sink` is a relevant data flow sink.
+   */
+  predicate isSink(Node sink);
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  default predicate isBarrier(Node node) { none() }
+
+  /** Holds if data flow into `node` is prohibited. */
+  default predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  default predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  default predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  default predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  default int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  default FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  default predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  default predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (as it is in a `path-problem` query).
+   */
+  default predicate includeHiddenNodes() { none() }
+}
+
+/** An input configuration for data flow using flow state. */
+signature module StateConfigSig {
+  bindingset[this]
+  class FlowState;
+
+  /**
+   * Holds if `source` is a relevant data flow source with the given initial
+   * `state`.
+   */
+  predicate isSource(Node source, FlowState state);
+
+  /**
+   * Holds if `sink` is a relevant data flow sink accepting `state`.
+   */
+  predicate isSink(Node sink, FlowState state);
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  default predicate isBarrier(Node node) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited when the flow state is
+   * `state`.
+   */
+  predicate isBarrier(Node node, FlowState state);
+
+  /** Holds if data flow into `node` is prohibited. */
+  default predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  default predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  default predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
+   */
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2);
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  default predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  default int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  default FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  default predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  default predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (as it is in a `path-problem` query).
+   */
+  default predicate includeHiddenNodes() { none() }
+}
+
+/**
+ * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
+ * measured in approximate number of interprocedural steps.
+ */
+signature int explorationLimitSig();
+
+/**
+ * The output of a data flow computation.
+ */
+signature module DataFlowSig {
+  /**
+   * A `Node` augmented with a call context (except for sinks) and an access path.
+   * Only those `PathNode`s that are reachable from a source, and which can reach a sink, are generated.
+   */
+  class PathNode;
+
+  /**
+   * Holds if data can flow from `source` to `sink`.
+   *
+   * The corresponding paths are generated from the end-points and the graph
+   * included in the module `PathGraph`.
+   */
+  predicate hasFlowPath(PathNode source, PathNode sink);
+
+  /**
+   * Holds if data can flow from `source` to `sink`.
+   */
+  predicate hasFlow(Node source, Node sink);
+
+  /**
+   * Holds if data can flow from some source to `sink`.
+   */
+  predicate hasFlowTo(Node sink);
+
+  /**
+   * Holds if data can flow from some source to `sink`.
+   */
+  predicate hasFlowToExpr(DataFlowExpr sink);
+}
+
+/**
+ * Constructs a standard data flow computation.
+ */
+module Make<ConfigSig Config> implements DataFlowSig {
+  private module C implements FullStateConfigSig {
+    import DefaultState<Config>
+    import Config
+  }
+
+  import Impl<C>
+}
+
+/**
+ * Constructs a data flow computation using flow state.
+ */
+module MakeWithState<StateConfigSig Config> implements DataFlowSig {
+  private module C implements FullStateConfigSig {
+    import Config
+  }
+
+  import Impl<C>
+}

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl1.qll

@@ -0,0 +1,396 @@
+/**
+ * DEPRECATED: Use `Make` and `MakeWithState` instead.
+ *
+ * Provides a `Configuration` class backwards-compatible interface to the data
+ * flow library.
+ */
+
+private import DataFlowImplCommon
+private import DataFlowImplSpecific::Private
+import DataFlowImplSpecific::Public
+private import DataFlowImpl
+import DataFlowImplCommonPublic
+import FlowStateString
+
+/**
+ * A configuration of interprocedural data flow analysis. This defines
+ * sources, sinks, and any other configurable aspect of the analysis. Each
+ * use of the global data flow library must define its own unique extension
+ * of this abstract class. To create a configuration, extend this class with
+ * a subclass whose characteristic predicate is a unique singleton string.
+ * For example, write
+ *
+ * ```ql
+ * class MyAnalysisConfiguration extends DataFlow::Configuration {
+ *   MyAnalysisConfiguration() { this = "MyAnalysisConfiguration" }
+ *   // Override `isSource` and `isSink`.
+ *   // Optionally override `isBarrier`.
+ *   // Optionally override `isAdditionalFlowStep`.
+ * }
+ * ```
+ * Conceptually, this defines a graph where the nodes are `DataFlow::Node`s and
+ * the edges are those data-flow steps that preserve the value of the node
+ * along with any additional edges defined by `isAdditionalFlowStep`.
+ * Specifying nodes in `isBarrier` will remove those nodes from the graph, and
+ * specifying nodes in `isBarrierIn` and/or `isBarrierOut` will remove in-going
+ * and/or out-going edges from those nodes, respectively.
+ *
+ * Then, to query whether there is flow between some `source` and `sink`,
+ * write
+ *
+ * ```ql
+ * exists(MyAnalysisConfiguration cfg | cfg.hasFlow(source, sink))
+ * ```
+ *
+ * Multiple configurations can coexist, but two classes extending
+ * `DataFlow::Configuration` should never depend on each other. One of them
+ * should instead depend on a `DataFlow2::Configuration`, a
+ * `DataFlow3::Configuration`, or a `DataFlow4::Configuration`.
+ */
+abstract class Configuration extends string {
+  bindingset[this]
+  Configuration() { any() }
+
+  /**
+   * Holds if `source` is a relevant data flow source.
+   */
+  predicate isSource(Node source) { none() }
+
+  /**
+   * Holds if `source` is a relevant data flow source with the given initial
+   * `state`.
+   */
+  predicate isSource(Node source, FlowState state) { none() }
+
+  /**
+   * Holds if `sink` is a relevant data flow sink.
+   */
+  predicate isSink(Node sink) { none() }
+
+  /**
+   * Holds if `sink` is a relevant data flow sink accepting `state`.
+   */
+  predicate isSink(Node sink, FlowState state) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  predicate isBarrier(Node node) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited when the flow state is
+   * `state`.
+   */
+  predicate isBarrier(Node node, FlowState state) { none() }
+
+  /** Holds if data flow into `node` is prohibited. */
+  predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * DEPRECATED: Use `isBarrier` and `BarrierGuard` module instead.
+   *
+   * Holds if data flow through nodes guarded by `guard` is prohibited.
+   */
+  deprecated predicate isBarrierGuard(BarrierGuard guard) { none() }
+
+  /**
+   * DEPRECATED: Use `isBarrier` and `BarrierGuard` module instead.
+   *
+   * Holds if data flow through nodes guarded by `guard` is prohibited when
+   * the flow state is `state`
+   */
+  deprecated predicate isBarrierGuard(BarrierGuard guard, FlowState state) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
+   */
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
+    none()
+  }
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if data may flow from `source` to `sink` for this configuration.
+   */
+  predicate hasFlow(Node source, Node sink) { hasFlow(source, sink, this) }
+
+  /**
+   * Holds if data may flow from `source` to `sink` for this configuration.
+   *
+   * The corresponding paths are generated from the end-points and the graph
+   * included in the module `PathGraph`.
+   */
+  predicate hasFlowPath(PathNode source, PathNode sink) { hasFlowPath(source, sink, this) }
+
+  /**
+   * Holds if data may flow from some source to `sink` for this configuration.
+   */
+  predicate hasFlowTo(Node sink) { hasFlowTo(sink, this) }
+
+  /**
+   * Holds if data may flow from some source to `sink` for this configuration.
+   */
+  predicate hasFlowToExpr(DataFlowExpr sink) { this.hasFlowTo(exprNode(sink)) }
+
+  /**
+   * DEPRECATED: Use `FlowExploration<explorationLimit>` instead.
+   *
+   * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
+   * measured in approximate number of interprocedural steps.
+   */
+  deprecated int explorationLimit() { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+}
+
+/**
+ * This class exists to prevent mutual recursion between the user-overridden
+ * member predicates of `Configuration` and the rest of the data-flow library.
+ * Good performance cannot be guaranteed in the presence of such recursion, so
+ * it should be replaced by using more than one copy of the data flow library.
+ */
+abstract private class ConfigurationRecursionPrevention extends Configuration {
+  bindingset[this]
+  ConfigurationRecursionPrevention() { any() }
+
+  override predicate hasFlow(Node source, Node sink) {
+    strictcount(Node n | this.isSource(n)) < 0
+    or
+    strictcount(Node n | this.isSource(n, _)) < 0
+    or
+    strictcount(Node n | this.isSink(n)) < 0
+    or
+    strictcount(Node n | this.isSink(n, _)) < 0
+    or
+    strictcount(Node n1, Node n2 | this.isAdditionalFlowStep(n1, n2)) < 0
+    or
+    strictcount(Node n1, Node n2 | this.isAdditionalFlowStep(n1, _, n2, _)) < 0
+    or
+    super.hasFlow(source, sink)
+  }
+}
+
+/** A bridge class to access the deprecated `isBarrierGuard`. */
+private class BarrierGuardGuardedNodeBridge extends Unit {
+  abstract predicate guardedNode(Node n, Configuration config);
+
+  abstract predicate guardedNode(Node n, FlowState state, Configuration config);
+}
+
+private class BarrierGuardGuardedNode extends BarrierGuardGuardedNodeBridge {
+  deprecated override predicate guardedNode(Node n, Configuration config) {
+    exists(BarrierGuard g |
+      config.isBarrierGuard(g) and
+      n = g.getAGuardedNode()
+    )
+  }
+
+  deprecated override predicate guardedNode(Node n, FlowState state, Configuration config) {
+    exists(BarrierGuard g |
+      config.isBarrierGuard(g, state) and
+      n = g.getAGuardedNode()
+    )
+  }
+}
+
+private FlowState relevantState(Configuration config) {
+  config.isSource(_, result) or
+  config.isSink(_, result) or
+  config.isBarrier(_, result) or
+  config.isAdditionalFlowStep(_, result, _, _) or
+  config.isAdditionalFlowStep(_, _, _, result)
+}
+
+private newtype TConfigState =
+  TMkConfigState(Configuration config, FlowState state) {
+    state = relevantState(config) or state instanceof FlowStateEmpty
+  }
+
+private Configuration getConfig(TConfigState state) { state = TMkConfigState(result, _) }
+
+private FlowState getState(TConfigState state) { state = TMkConfigState(_, result) }
+
+private predicate singleConfiguration() { 1 = strictcount(Configuration c) }
+
+private module Config implements FullStateConfigSig {
+  class FlowState = TConfigState;
+
+  predicate isSource(Node source, FlowState state) {
+    getConfig(state).isSource(source, getState(state))
+    or
+    getConfig(state).isSource(source) and getState(state) instanceof FlowStateEmpty
+  }
+
+  predicate isSink(Node sink, FlowState state) {
+    getConfig(state).isSink(sink, getState(state))
+    or
+    getConfig(state).isSink(sink) and getState(state) instanceof FlowStateEmpty
+  }
+
+  predicate isBarrier(Node node) { none() }
+
+  predicate isBarrier(Node node, FlowState state) {
+    getConfig(state).isBarrier(node, getState(state)) or
+    getConfig(state).isBarrier(node) or
+    any(BarrierGuardGuardedNodeBridge b).guardedNode(node, getState(state), getConfig(state)) or
+    any(BarrierGuardGuardedNodeBridge b).guardedNode(node, getConfig(state))
+  }
+
+  predicate isBarrierIn(Node node) { any(Configuration config).isBarrierIn(node) }
+
+  predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
+
+  predicate isAdditionalFlowStep(Node node1, Node node2) {
+    singleConfiguration() and
+    any(Configuration config).isAdditionalFlowStep(node1, node2)
+  }
+
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
+    getConfig(state1).isAdditionalFlowStep(node1, getState(state1), node2, getState(state2)) and
+    getConfig(state2) = getConfig(state1)
+    or
+    not singleConfiguration() and
+    getConfig(state1).isAdditionalFlowStep(node1, node2) and
+    state2 = state1
+  }
+
+  predicate allowImplicitRead(Node node, ContentSet c) {
+    any(Configuration config).allowImplicitRead(node, c)
+  }
+
+  int fieldFlowBranchLimit() { result = min(any(Configuration config).fieldFlowBranchLimit()) }
+
+  FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
+
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
+  predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
+}
+
+private import Impl<Config> as I
+import I
+
+/**
+ * A `Node` augmented with a call context (except for sinks), an access path, and a configuration.
+ * Only those `PathNode`s that are reachable from a source, and which can reach a sink, are generated.
+ */
+class PathNode instanceof I::PathNode {
+  /** Gets a textual representation of this element. */
+  final string toString() { result = super.toString() }
+
+  /**
+   * Gets a textual representation of this element, including a textual
+   * representation of the call context.
+   */
+  final string toStringWithContext() { result = super.toStringWithContext() }
+
+  /**
+   * Holds if this element is at the specified location.
+   * The location spans column `startcolumn` of line `startline` to
+   * column `endcolumn` of line `endline` in file `filepath`.
+   * For more information, see
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
+   */
+  final predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    super.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+  }
+
+  /** Gets the underlying `Node`. */
+  final Node getNode() { result = super.getNode() }
+
+  /** Gets the `FlowState` of this node. */
+  final FlowState getState() { result = getState(super.getState()) }
+
+  /** Gets the associated configuration. */
+  final Configuration getConfiguration() { result = getConfig(super.getState()) }
+
+  /** Gets a successor of this node, if any. */
+  final PathNode getASuccessor() { result = super.getASuccessor() }
+
+  /** Holds if this node is a source. */
+  final predicate isSource() { super.isSource() }
+
+  /** Holds if this node is a grouping of source nodes. */
+  final predicate isSourceGroup(string group) { super.isSourceGroup(group) }
+
+  /** Holds if this node is a grouping of sink nodes. */
+  final predicate isSinkGroup(string group) { super.isSinkGroup(group) }
+}
+
+private predicate hasFlow(Node source, Node sink, Configuration config) {
+  exists(PathNode source0, PathNode sink0 |
+    hasFlowPath(source0, sink0, config) and
+    source0.getNode() = source and
+    sink0.getNode() = sink
+  )
+}
+
+private predicate hasFlowPath(PathNode source, PathNode sink, Configuration config) {
+  hasFlowPath(source, sink) and source.getConfiguration() = config
+}
+
+private predicate hasFlowTo(Node sink, Configuration config) { hasFlow(_, sink, config) }
+
+predicate flowsTo = hasFlow/3;

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll

@@ -3,6 +3,8 @@ private import DataFlowImplSpecific::Public
 import Cached
 
 module DataFlowImplCommonPublic {
+  /** Provides `FlowState = string`. */
+  module FlowStateString {
     /** A state value to track during data flow. */
     class FlowState = string;
 
@@ -13,6 +15,7 @@ module DataFlowImplCommonPublic {
     class FlowStateEmpty extends FlowState {
       FlowStateEmpty() { this = "" }
     }
+  }
 
   private newtype TFlowFeature =
     TFeatureHasSourceCallContext() or

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll

@@ -414,3 +414,12 @@ private class MyConsistencyConfiguration extends Consistency::ConsistencyConfigu
     any()
   }
 }
+
+/**
+ * Gets an additional term that is added to the `join` and `branch` computations to reflect
+ * an additional forward or backwards branching factor that is not taken into account
+ * when calculating the (virtual) dispatch cost.
+ *
+ * Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter.
+ */
+int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p) { none() }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/PrintIRLocalFlow.qll

@@ -97,23 +97,23 @@ private string getNodeProperty(DataFlow::Node node, string key) {
     |
       kind, ", "
     )
-  or
+  // or
-  // Is there partial flow from a source to this node?
+  // // Is there partial flow from a source to this node?
-  // This property will only be emitted if partial flow is enabled by overriding
+  // // This property will only be emitted if partial flow is enabled by overriding
-  // `DataFlow::Configuration::explorationLimit()`.
+  // // `DataFlow::Configuration::explorationLimit()`.
-  key = "pflow" and
+  // key = "pflow" and
-  result =
+  // result =
-    strictconcat(DataFlow::PartialPathNode sourceNode, DataFlow::PartialPathNode destNode, int dist,
+  //   strictconcat(DataFlow::PartialPathNode sourceNode, DataFlow::PartialPathNode destNode, int dist,
-      int order1, int order2 |
+  //     int order1, int order2 |
-      any(DataFlow::Configuration cfg).hasPartialFlow(sourceNode, destNode, dist) and
+  //     any(DataFlow::Configuration cfg).hasPartialFlow(sourceNode, destNode, dist) and
-      destNode.getNode() = node and
+  //     destNode.getNode() = node and
-      // Only print flow from a source in the same function.
+  //     // Only print flow from a source in the same function.
-      sourceNode.getNode().getEnclosingCallable() = node.getEnclosingCallable()
+  //     sourceNode.getNode().getEnclosingCallable() = node.getEnclosingCallable()
-    |
+  //   |
-      nodeId(sourceNode.getNode(), order1, order2) + "+" + dist.toString(), ", "
+  //     nodeId(sourceNode.getNode(), order1, order2) + "+" + dist.toString(), ", "
-      order by
+  //     order by
-        order1, order2, dist desc
+  //       order1, order2, dist desc
-    )
+  //   )
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTracking.qll

@@ -0,0 +1,63 @@
+/**
+ * Provides classes for performing local (intra-procedural) and
+ * global (inter-procedural) taint-tracking analyses.
+ */
+
+import TaintTrackingParameter::Public
+private import TaintTrackingParameter::Private
+
+private module AddTaintDefaults<DataFlowInternal::FullStateConfigSig Config> implements
+DataFlowInternal::FullStateConfigSig {
+  import Config
+
+  predicate isBarrier(DataFlow::Node node) {
+    Config::isBarrier(node) or defaultTaintSanitizer(node)
+  }
+
+  predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
+    Config::isAdditionalFlowStep(node1, node2) or
+    defaultAdditionalTaintStep(node1, node2)
+  }
+
+  predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c) {
+    Config::allowImplicitRead(node, c)
+    or
+    (
+      Config::isSink(node, _) or
+      Config::isAdditionalFlowStep(node, _) or
+      Config::isAdditionalFlowStep(node, _, _, _)
+    ) and
+    defaultImplicitTaintRead(node, c)
+  }
+}
+
+/**
+ * Constructs a standard taint tracking computation.
+ */
+module Make<DataFlow::ConfigSig Config> implements DataFlow::DataFlowSig {
+  private module Config0 implements DataFlowInternal::FullStateConfigSig {
+    import DataFlowInternal::DefaultState<Config>
+    import Config
+  }
+
+  private module C implements DataFlowInternal::FullStateConfigSig {
+    import AddTaintDefaults<Config0>
+  }
+
+  import DataFlowInternal::Impl<C>
+}
+
+/**
+ * Constructs a taint tracking computation using flow state.
+ */
+module MakeWithState<DataFlow::StateConfigSig Config> implements DataFlow::DataFlowSig {
+  private module Config0 implements DataFlowInternal::FullStateConfigSig {
+    import Config
+  }
+
+  private module C implements DataFlowInternal::FullStateConfigSig {
+    import AddTaintDefaults<Config0>
+  }
+
+  import DataFlowInternal::Impl<C>
+}

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingParameter.qll

@@ -2,4 +2,5 @@ import semmle.code.cpp.ir.dataflow.internal.TaintTrackingUtil as Public
 
 module Private {
   import semmle.code.cpp.ir.dataflow.DataFlow::DataFlow as DataFlow
+  import semmle.code.cpp.ir.dataflow.internal.DataFlowImpl as DataFlowInternal
 }

cpp/ql/src/Security/CWE/CWE-078/ExecTainted.ql

@@ -22,7 +22,7 @@ import semmle.code.cpp.ir.dataflow.TaintTracking
 import semmle.code.cpp.ir.dataflow.TaintTracking2
 import semmle.code.cpp.security.FlowSources
 import semmle.code.cpp.models.implementations.Strcat
-import DataFlow::PathGraph
+import ExecTaint::PathGraph
 
 Expr sinkAsArgumentIndirection(DataFlow::Node sink) {
   result =
@@ -67,28 +67,28 @@ predicate interestingConcatenation(DataFlow::Node fst, DataFlow::Node snd) {
   )
 }
 
-class ConcatState extends DataFlow::FlowState {
+newtype TState =
-  ConcatState() { this = "ConcatState" }
+  TConcatState() or
+  TExecState(DataFlow::Node fst, DataFlow::Node snd) { interestingConcatenation(fst, snd) }
+
+class ConcatState extends TConcatState {
+  string toString() { result = "ConcatState" }
 }
 
-class ExecState extends DataFlow::FlowState {
+class ExecState extends TExecState {
   DataFlow::Node fst;
   DataFlow::Node snd;
 
-  ExecState() {
+  ExecState() { this = TExecState(fst, snd) }
-    this =
-      "ExecState (" + fst.getLocation() + " | " + fst + ", " + snd.getLocation() + " | " + snd + ")" and
-    interestingConcatenation(pragma[only_bind_into](fst), pragma[only_bind_into](snd))
-  }
 
   DataFlow::Node getFstNode() { result = fst }
 
   DataFlow::Node getSndNode() { result = snd }
 
   /** Holds if this is a possible `ExecState` for `sink`. */
-  predicate isFeasibleForSink(DataFlow::Node sink) {
+  predicate isFeasibleForSink(DataFlow::Node sink) { ExecState::hasFlow(snd, sink) }
-    any(ExecStateConfiguration conf).hasFlow(snd, sink)
+
-  }
+  string toString() { result = "ExecState" }
 }
 
 /**
@@ -96,45 +96,42 @@ class ExecState extends DataFlow::FlowState {
  * given sink. This avoids a cartesian product between all sinks and all `ExecState`s in
  * `ExecTaintConfiguration::isSink`.
  */
-class ExecStateConfiguration extends TaintTracking2::Configuration {
+module ExecStateConfiguration implements DataFlow::ConfigSig {
-  ExecStateConfiguration() { this = "ExecStateConfiguration" }
+  predicate isSource(DataFlow::Node source) {
-
-  override predicate isSource(DataFlow::Node source) {
     exists(ExecState state | state.getSndNode() = source)
   }
 
-  override predicate isSink(DataFlow::Node sink) {
+  predicate isSink(DataFlow::Node sink) { shellCommand(sinkAsArgumentIndirection(sink), _) }
-    shellCommand(sinkAsArgumentIndirection(sink), _)
-  }
 
-  override predicate isSanitizerOut(DataFlow::Node node) {
+  predicate isBarrierOut(DataFlow::Node node) {
-    isSink(node, _) // Prevent duplicates along a call chain, since `shellCommand` will include wrappers
+    isSink(node) // Prevent duplicates along a call chain, since `shellCommand` will include wrappers
   }
 }
 
-class ExecTaintConfiguration extends TaintTracking::Configuration {
+module ExecState = TaintTracking::Make<ExecStateConfiguration>;
-  ExecTaintConfiguration() { this = "ExecTaintConfiguration" }
 
-  override predicate isSource(DataFlow::Node source, DataFlow::FlowState state) {
+module ExecTaintConfiguration implements DataFlow::StateConfigSig {
+  class FlowState = TState;
+
+  predicate isSource(DataFlow::Node source, FlowState state) {
     source instanceof FlowSource and
     state instanceof ConcatState
   }
 
-  override predicate isSink(DataFlow::Node sink, DataFlow::FlowState state) {
+  predicate isSink(DataFlow::Node sink, FlowState state) {
-    any(ExecStateConfiguration conf).isSink(sink) and
+    ExecStateConfiguration::isSink(sink) and
     state.(ExecState).isFeasibleForSink(sink)
   }
 
-  override predicate isAdditionalTaintStep(
+  predicate isAdditionalFlowStep(
-    DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,
+    DataFlow::Node node1, FlowState state1, DataFlow::Node node2, FlowState state2
-    DataFlow::FlowState state2
   ) {
     state1 instanceof ConcatState and
     state2.(ExecState).getFstNode() = node1 and
     state2.(ExecState).getSndNode() = node2
   }
 
-  override predicate isSanitizer(DataFlow::Node node, DataFlow::FlowState state) {
+  predicate isBarrier(DataFlow::Node node, FlowState state) {
     (
       node.asInstruction().getResultType() instanceof IntegralType
       or
@@ -143,16 +140,18 @@ class ExecTaintConfiguration extends TaintTracking::Configuration {
     state instanceof ConcatState
   }
 
-  override predicate isSanitizerOut(DataFlow::Node node) {
+  predicate isBarrierOut(DataFlow::Node node) {
     isSink(node, _) // Prevent duplicates along a call chain, since `shellCommand` will include wrappers
   }
 }
 
+module ExecTaint = TaintTracking::MakeWithState<ExecTaintConfiguration>;
+
 from
-  ExecTaintConfiguration conf, DataFlow::PathNode sourceNode, DataFlow::PathNode sinkNode,
+  ExecTaint::PathNode sourceNode, ExecTaint::PathNode sinkNode, string taintCause, string callChain,
-  string taintCause, string callChain, DataFlow::Node concatResult
+  DataFlow::Node concatResult
 where
-  conf.hasFlowPath(sourceNode, sinkNode) and
+  ExecTaint::hasFlowPath(sourceNode, sinkNode) and
   taintCause = sourceNode.getNode().(FlowSource).getSourceType() and
   shellCommand(sinkAsArgumentIndirection(sinkNode.getNode()), callChain) and
   concatResult = sinkNode.getState().(ExecState).getSndNode()

cpp/ql/src/Security/CWE/CWE-732/DoNotCreateWorldWritable.c

@@ -1,11 +1,11 @@
-int write_default_config_bad() {
+void write_default_config_bad() {
 	// BAD - this is world-writable so any user can overwrite the config
-	FILE* out = creat(OUTFILE, 0666);
+	int out = creat(OUTFILE, 0666);
-	fprintf(out, DEFAULT_CONFIG);
+	dprintf(out, DEFAULT_CONFIG);
 }
 
-int write_default_config_good() {
+void write_default_config_good() {
 	// GOOD - this allows only the current user to modify the file
-	FILE* out = creat(OUTFILE, S_IWUSR | S_IRUSR);
+	int out = creat(OUTFILE, S_IWUSR | S_IRUSR);
-	fprintf(out, DEFAULT_CONFIG);
+	dprintf(out, DEFAULT_CONFIG);
 }

cpp/ql/src/experimental/Security/CWE/CWE-415/DoubleFree.ql

@@ -14,8 +14,8 @@ import cpp
 
 from FunctionCall fc, FunctionCall fc2, LocalScopeVariable v
 where
-  freeCall(fc, v.getAnAccess()) and
+  fc.(DeallocationExpr).getFreedExpr() = v.getAnAccess() and
-  freeCall(fc2, v.getAnAccess()) and
+  fc2.(DeallocationExpr).getFreedExpr() = v.getAnAccess() and
   fc != fc2 and
   fc.getASuccessor*() = fc2 and
   not exists(Expr exptmp |

cpp/ql/test/library-tests/CPP-205/elements.expected

@@ -1,14 +1,14 @@
 | CPP-205.cpp:0:0:0:0 | CPP-205.cpp |  |
 | CPP-205.cpp:1:20:1:20 | T |  |
 | CPP-205.cpp:1:20:1:20 | definition of T |  |
-| CPP-205.cpp:2:5:2:5 | definition of fn | function declaration entry for fn<int>(int) -> int |
+| CPP-205.cpp:2:5:2:5 | definition of fn | function declaration entry for int fn<int>(int) |
-| CPP-205.cpp:2:5:2:5 | fn | function fn<int>(int) -> int |
+| CPP-205.cpp:2:5:2:5 | fn | function int fn<int>(int) |
-| CPP-205.cpp:2:5:2:6 | definition of fn | function declaration entry for fn<T>(T) -> int |
+| CPP-205.cpp:2:5:2:6 | definition of fn | function declaration entry for int fn<T>(T) |
-| CPP-205.cpp:2:5:2:6 | fn | function fn<T>(T) -> int |
+| CPP-205.cpp:2:5:2:6 | fn | function int fn<T>(T) |
-| CPP-205.cpp:2:10:2:12 | definition of out | parameter declaration entry for fn<T>(T) -> int |
+| CPP-205.cpp:2:10:2:12 | definition of out | parameter declaration entry for int fn<T>(T) |
-| CPP-205.cpp:2:10:2:12 | definition of out | parameter declaration entry for fn<int>(int) -> int |
+| CPP-205.cpp:2:10:2:12 | definition of out | parameter declaration entry for int fn<int>(int) |
-| CPP-205.cpp:2:10:2:12 | out | parameter for fn<T>(T) -> int |
+| CPP-205.cpp:2:10:2:12 | out | parameter for int fn<T>(T) |
-| CPP-205.cpp:2:10:2:12 | out | parameter for fn<int>(int) -> int |
+| CPP-205.cpp:2:10:2:12 | out | parameter for int fn<int>(int) |
 | CPP-205.cpp:2:15:5:1 | { ... } |  |
 | CPP-205.cpp:2:15:5:1 | { ... } |  |
 | CPP-205.cpp:3:3:3:33 | declaration |  |
@@ -20,16 +20,16 @@
 | CPP-205.cpp:4:3:4:11 | return ... |  |
 | CPP-205.cpp:4:10:4:10 | 0 |  |
 | CPP-205.cpp:4:10:4:10 | 0 |  |
-| CPP-205.cpp:7:5:7:8 | definition of main | function declaration entry for main() -> int |
+| CPP-205.cpp:7:5:7:8 | definition of main | function declaration entry for int main() |
-| CPP-205.cpp:7:5:7:8 | main | function main() -> int |
+| CPP-205.cpp:7:5:7:8 | main | function int main() |
 | CPP-205.cpp:7:12:9:1 | { ... } |  |
 | CPP-205.cpp:8:3:8:15 | return ... |  |
 | CPP-205.cpp:8:10:8:11 | call to fn |  |
 | CPP-205.cpp:8:13:8:13 | 0 |  |
-| file://:0:0:0:0 | (unnamed parameter 0) | parameter for __va_list_tag::operator=(__va_list_tag &&) -> __va_list_tag & |
+| file://:0:0:0:0 | (unnamed parameter 0) | parameter for __va_list_tag& __va_list_tag::operator=(__va_list_tag const&) |
-| file://:0:0:0:0 | (unnamed parameter 0) | parameter for __va_list_tag::operator=(const __va_list_tag &) -> __va_list_tag & |
+| file://:0:0:0:0 | (unnamed parameter 0) | parameter for __va_list_tag& __va_list_tag::operator=(__va_list_tag&&) |
 | file://:0:0:0:0 | __super |  |
 | file://:0:0:0:0 | __va_list_tag |  |
-| file://:0:0:0:0 | operator= | function __va_list_tag::operator=(__va_list_tag &&) -> __va_list_tag & |
+| file://:0:0:0:0 | operator= | function __va_list_tag& __va_list_tag::operator=(__va_list_tag const&) |
-| file://:0:0:0:0 | operator= | function __va_list_tag::operator=(const __va_list_tag &) -> __va_list_tag & |
+| file://:0:0:0:0 | operator= | function __va_list_tag& __va_list_tag::operator=(__va_list_tag&&) |
 | file://:0:0:0:0 | y |  |

cpp/ql/test/library-tests/CPP-205/elements.ql

@@ -1,17 +1,19 @@
 import cpp
+import semmle.code.cpp.Print
 
 string describe(Element e) {
-  result = "function " + e.(Function).getFullSignature()
+  e instanceof Function and
+  result = "function " + getIdentityString(e)
   or
   result =
     "function declaration entry for " +
-      e.(FunctionDeclarationEntry).getFunction().getFullSignature()
+      getIdentityString(e.(FunctionDeclarationEntry).getFunction())
   or
-  result = "parameter for " + e.(Parameter).getFunction().getFullSignature()
+  result = "parameter for " + getIdentityString(e.(Parameter).getFunction())
   or
   result =
     "parameter declaration entry for " +
-      e.(ParameterDeclarationEntry).getFunctionDeclarationEntry().getFunction().getFullSignature()
+      getIdentityString(e.(ParameterDeclarationEntry).getFunctionDeclarationEntry().getFunction())
 }
 
 from Element e

cpp/ql/test/library-tests/allocators/allocators.expected

@@ -1,51 +1,51 @@
 newExprs
-| allocators.cpp:49:3:49:9 | new | int | operator new(unsigned long) -> void * | 4 | 4 |  |  |
+| allocators.cpp:49:3:49:9 | new | int | void* operator new(unsigned long) | 4 | 4 |  |  |
-| allocators.cpp:50:3:50:15 | new | int | operator new(size_t, float) -> void * | 4 | 4 |  |  |
+| allocators.cpp:50:3:50:15 | new | int | void* operator new(size_t, float) | 4 | 4 |  |  |
-| allocators.cpp:51:3:51:11 | new | int | operator new(unsigned long) -> void * | 4 | 4 |  |  |
+| allocators.cpp:51:3:51:11 | new | int | void* operator new(unsigned long) | 4 | 4 |  |  |
-| allocators.cpp:52:3:52:14 | new | String | operator new(unsigned long) -> void * | 8 | 8 |  |  |
+| allocators.cpp:52:3:52:14 | new | String | void* operator new(unsigned long) | 8 | 8 |  |  |
-| allocators.cpp:53:3:53:27 | new | String | operator new(size_t, float) -> void * | 8 | 8 |  |  |
+| allocators.cpp:53:3:53:27 | new | String | void* operator new(size_t, float) | 8 | 8 |  |  |
-| allocators.cpp:54:3:54:17 | new | Overaligned | operator new(unsigned long, align_val_t) -> void * | 256 | 128 | aligned |  |
+| allocators.cpp:54:3:54:17 | new | Overaligned | void* operator new(unsigned long, std::align_val_t) | 256 | 128 | aligned |  |
-| allocators.cpp:55:3:55:25 | new | Overaligned | operator new(size_t, align_val_t, float) -> void * | 256 | 128 | aligned |  |
+| allocators.cpp:55:3:55:25 | new | Overaligned | void* operator new(size_t, std::align_val_t, float) | 256 | 128 | aligned |  |
-| allocators.cpp:107:3:107:18 | new | FailedInit | FailedInit::operator new(size_t) -> void * | 1 | 1 |  |  |
+| allocators.cpp:107:3:107:18 | new | FailedInit | void* FailedInit::operator new(size_t) | 1 | 1 |  |  |
-| allocators.cpp:109:3:109:35 | new | FailedInitOveraligned | FailedInitOveraligned::operator new(size_t, align_val_t, float) -> void * | 128 | 128 | aligned |  |
+| allocators.cpp:109:3:109:35 | new | FailedInitOveraligned | void* FailedInitOveraligned::operator new(size_t, std::align_val_t, float) | 128 | 128 | aligned |  |
-| allocators.cpp:129:3:129:21 | new | int | operator new(size_t, void *) -> void * | 4 | 4 |  | & ... |
+| allocators.cpp:129:3:129:21 | new | int | void* operator new(std::size_t, void*) | 4 | 4 |  | & ... |
-| allocators.cpp:135:3:135:26 | new | int | operator new(size_t, const nothrow_t &) -> void * | 4 | 4 |  |  |
+| allocators.cpp:135:3:135:26 | new | int | void* operator new(std::size_t, std::nothrow_t const&) | 4 | 4 |  |  |
 newArrayExprs
-| allocators.cpp:68:3:68:12 | new[] | int[] | int | operator new[](unsigned long) -> void * | 4 | 4 |  | n |  |
+| allocators.cpp:68:3:68:12 | new[] | int[] | int | void* operator new[](unsigned long) | 4 | 4 |  | n |  |
-| allocators.cpp:69:3:69:18 | new[] | int[] | int | operator new[](size_t, float) -> void * | 4 | 4 |  | n |  |
+| allocators.cpp:69:3:69:18 | new[] | int[] | int | void* operator new[](size_t, float) | 4 | 4 |  | n |  |
-| allocators.cpp:70:3:70:15 | new[] | String[] | String | operator new[](unsigned long) -> void * | 8 | 8 |  | n |  |
+| allocators.cpp:70:3:70:15 | new[] | String[] | String | void* operator new[](unsigned long) | 8 | 8 |  | n |  |
-| allocators.cpp:71:3:71:20 | new[] | Overaligned[] | Overaligned | operator new[](unsigned long, align_val_t) -> void * | 256 | 128 | aligned | n |  |
+| allocators.cpp:71:3:71:20 | new[] | Overaligned[] | Overaligned | void* operator new[](unsigned long, std::align_val_t) | 256 | 128 | aligned | n |  |
-| allocators.cpp:72:3:72:16 | new[] | String[10] | String | operator new[](unsigned long) -> void * | 8 | 8 |  |  |  |
+| allocators.cpp:72:3:72:16 | new[] | String[10] | String | void* operator new[](unsigned long) | 8 | 8 |  |  |  |
-| allocators.cpp:108:3:108:19 | new[] | FailedInit[] | FailedInit | FailedInit::operator new[](size_t) -> void * | 1 | 1 |  | n |  |
+| allocators.cpp:108:3:108:19 | new[] | FailedInit[] | FailedInit | void* FailedInit::operator new[](size_t) | 1 | 1 |  | n |  |
-| allocators.cpp:110:3:110:37 | new[] | FailedInitOveraligned[10] | FailedInitOveraligned | FailedInitOveraligned::operator new[](size_t, align_val_t, float) -> void * | 128 | 128 | aligned |  |  |
+| allocators.cpp:110:3:110:37 | new[] | FailedInitOveraligned[10] | FailedInitOveraligned | void* FailedInitOveraligned::operator new[](size_t, std::align_val_t, float) | 128 | 128 | aligned |  |  |
-| allocators.cpp:132:3:132:17 | new[] | int[1] | int | operator new[](size_t, void *) -> void * | 4 | 4 |  |  | buf |
+| allocators.cpp:132:3:132:17 | new[] | int[1] | int | void* operator new[](std::size_t, void*) | 4 | 4 |  |  | buf |
-| allocators.cpp:136:3:136:26 | new[] | int[2] | int | operator new[](size_t, const nothrow_t &) -> void * | 4 | 4 |  |  |  |
+| allocators.cpp:136:3:136:26 | new[] | int[2] | int | void* operator new[](std::size_t, std::nothrow_t const&) | 4 | 4 |  |  |  |
-| allocators.cpp:142:13:142:27 | new[] | char[][10] | char[10] | operator new[](unsigned long) -> void * | 10 | 1 |  | x |  |
+| allocators.cpp:142:13:142:27 | new[] | char[][10] | char[10] | void* operator new[](unsigned long) | 10 | 1 |  | x |  |
-| allocators.cpp:143:13:143:28 | new[] | char[20][20] | char[20] | operator new[](unsigned long) -> void * | 20 | 1 |  |  |  |
+| allocators.cpp:143:13:143:28 | new[] | char[20][20] | char[20] | void* operator new[](unsigned long) | 20 | 1 |  |  |  |
-| allocators.cpp:144:13:144:31 | new[] | char[][30][30] | char[30][30] | operator new[](unsigned long) -> void * | 900 | 1 |  | x |  |
+| allocators.cpp:144:13:144:31 | new[] | char[][30][30] | char[30][30] | void* operator new[](unsigned long) | 900 | 1 |  | x |  |
 newExprDeallocators
-| allocators.cpp:52:3:52:14 | new | String | operator delete(void *, unsigned long) -> void | 8 | 8 | sized  |
+| allocators.cpp:52:3:52:14 | new | String | void operator delete(void*, unsigned long) | 8 | 8 | sized  |
-| allocators.cpp:53:3:53:27 | new | String | operator delete(void *, float) -> void | 8 | 8 |   |
+| allocators.cpp:53:3:53:27 | new | String | void operator delete(void*, float) | 8 | 8 |   |
-| allocators.cpp:107:3:107:18 | new | FailedInit | FailedInit::operator delete(void *, size_t) -> void | 1 | 1 | sized  |
+| allocators.cpp:107:3:107:18 | new | FailedInit | void FailedInit::operator delete(void*, size_t) | 1 | 1 | sized  |
-| allocators.cpp:109:3:109:35 | new | FailedInitOveraligned | FailedInitOveraligned::operator delete(void *, align_val_t, float) -> void | 128 | 128 |  aligned |
+| allocators.cpp:109:3:109:35 | new | FailedInitOveraligned | void FailedInitOveraligned::operator delete(void*, std::align_val_t, float) | 128 | 128 |  aligned |
 newArrayExprDeallocators
-| allocators.cpp:70:3:70:15 | new[] | String | operator delete[](void *, unsigned long) -> void | 8 | 8 | sized  |
+| allocators.cpp:70:3:70:15 | new[] | String | void operator delete[](void*, unsigned long) | 8 | 8 | sized  |
-| allocators.cpp:72:3:72:16 | new[] | String | operator delete[](void *, unsigned long) -> void | 8 | 8 | sized  |
+| allocators.cpp:72:3:72:16 | new[] | String | void operator delete[](void*, unsigned long) | 8 | 8 | sized  |
-| allocators.cpp:108:3:108:19 | new[] | FailedInit | FailedInit::operator delete[](void *, size_t) -> void | 1 | 1 | sized  |
+| allocators.cpp:108:3:108:19 | new[] | FailedInit | void FailedInit::operator delete[](void*, size_t) | 1 | 1 | sized  |
-| allocators.cpp:110:3:110:37 | new[] | FailedInitOveraligned | FailedInitOveraligned::operator delete[](void *, align_val_t, float) -> void | 128 | 128 |  aligned |
+| allocators.cpp:110:3:110:37 | new[] | FailedInitOveraligned | void FailedInitOveraligned::operator delete[](void*, std::align_val_t, float) | 128 | 128 |  aligned |
 deleteExprs
-| allocators.cpp:59:3:59:35 | delete | int | operator delete(void *, unsigned long) -> void | 4 | 4 | sized  |
+| allocators.cpp:59:3:59:35 | delete | int | void operator delete(void*, unsigned long) | 4 | 4 | sized  |
-| allocators.cpp:60:3:60:38 | delete | String | operator delete(void *, unsigned long) -> void | 8 | 8 | sized  |
+| allocators.cpp:60:3:60:38 | delete | String | void operator delete(void*, unsigned long) | 8 | 8 | sized  |
-| allocators.cpp:61:3:61:44 | delete | SizedDealloc | SizedDealloc::operator delete(void *, size_t) -> void | 32 | 1 | sized  |
+| allocators.cpp:61:3:61:44 | delete | SizedDealloc | void SizedDealloc::operator delete(void*, size_t) | 32 | 1 | sized  |
-| allocators.cpp:62:3:62:43 | delete | Overaligned | operator delete(void *, unsigned long, align_val_t) -> void | 256 | 128 | sized aligned |
+| allocators.cpp:62:3:62:43 | delete | Overaligned | void operator delete(void*, unsigned long, std::align_val_t) | 256 | 128 | sized aligned |
-| allocators.cpp:64:3:64:44 | delete | const String | operator delete(void *, unsigned long) -> void | 8 | 8 | sized  |
+| allocators.cpp:64:3:64:44 | delete | const String | void operator delete(void*, unsigned long) | 8 | 8 | sized  |
 deleteArrayExprs
-| allocators.cpp:78:3:78:37 | delete[] | int | operator delete[](void *, unsigned long) -> void | 4 | 4 | sized  |
+| allocators.cpp:78:3:78:37 | delete[] | int | void operator delete[](void*, unsigned long) | 4 | 4 | sized  |
-| allocators.cpp:79:3:79:40 | delete[] | String | operator delete[](void *, unsigned long) -> void | 8 | 8 | sized  |
+| allocators.cpp:79:3:79:40 | delete[] | String | void operator delete[](void*, unsigned long) | 8 | 8 | sized  |
-| allocators.cpp:80:3:80:46 | delete[] | SizedDealloc | SizedDealloc::operator delete[](void *, size_t) -> void | 32 | 1 | sized  |
+| allocators.cpp:80:3:80:46 | delete[] | SizedDealloc | void SizedDealloc::operator delete[](void*, size_t) | 32 | 1 | sized  |
-| allocators.cpp:81:3:81:45 | delete[] | Overaligned | operator delete[](void *, unsigned long, align_val_t) -> void | 256 | 128 | sized aligned |
+| allocators.cpp:81:3:81:45 | delete[] | Overaligned | void operator delete[](void*, unsigned long, std::align_val_t) | 256 | 128 | sized aligned |
-| allocators.cpp:82:3:82:49 | delete[] | PolymorphicBase | operator delete[](void *, unsigned long) -> void | 8 | 8 | sized  |
+| allocators.cpp:82:3:82:49 | delete[] | PolymorphicBase | void operator delete[](void*, unsigned long) | 8 | 8 | sized  |
-| allocators.cpp:83:3:83:23 | delete[] | int | operator delete[](void *, unsigned long) -> void | 4 | 4 | sized  |
+| allocators.cpp:83:3:83:23 | delete[] | int | void operator delete[](void*, unsigned long) | 4 | 4 | sized  |
 allocationFunctions
 | allocators.cpp:7:7:7:18 | operator new | getSizeArg = 0, requiresDealloc |
 | allocators.cpp:8:7:8:20 | operator new[] | getSizeArg = 0, requiresDealloc |

cpp/ql/test/library-tests/allocators/allocators.ql

@@ -1,12 +1,13 @@
 import cpp
 import semmle.code.cpp.models.implementations.Allocation
+import semmle.code.cpp.Print
 
 query predicate newExprs(
   NewExpr expr, string type, string sig, int size, int alignment, string form, string placement
 ) {
   exists(Function allocator, Type allocatedType |
     expr.getAllocator() = allocator and
-    sig = allocator.getFullSignature() and
+    sig = getIdentityString(allocator) and
     allocatedType = expr.getAllocatedType() and
     type = allocatedType.toString() and
     size = allocatedType.getSize() and
@@ -24,7 +25,7 @@ query predicate newArrayExprs(
 ) {
   exists(Function allocator, Type arrayType, Type elementType |
     expr.getAllocator() = allocator and
-    sig = allocator.getFullSignature() and
+    sig = getIdentityString(allocator) and
     arrayType = expr.getAllocatedType() and
     t1 = arrayType.toString() and
     elementType = expr.getAllocatedElementType() and
@@ -44,7 +45,7 @@ query predicate newExprDeallocators(
 ) {
   exists(Function deallocator, Type allocatedType |
     expr.getDeallocator() = deallocator and
-    sig = deallocator.getFullSignature() and
+    sig = getIdentityString(deallocator) and
     allocatedType = expr.getAllocatedType() and
     type = allocatedType.toString() and
     size = allocatedType.getSize() and
@@ -62,7 +63,7 @@ query predicate newArrayExprDeallocators(
 ) {
   exists(Function deallocator, Type elementType |
     expr.getDeallocator() = deallocator and
-    sig = deallocator.getFullSignature() and
+    sig = getIdentityString(deallocator) and
     elementType = expr.getAllocatedElementType() and
     type = elementType.toString() and
     size = elementType.getSize() and
@@ -80,7 +81,7 @@ query predicate deleteExprs(
 ) {
   exists(Function deallocator, Type deletedType |
     expr.getDeallocator() = deallocator and
-    sig = deallocator.getFullSignature() and
+    sig = getIdentityString(deallocator) and
     deletedType = expr.getDeletedObjectType() and
     type = deletedType.toString() and
     size = deletedType.getSize() and
@@ -98,7 +99,7 @@ query predicate deleteArrayExprs(
 ) {
   exists(Function deallocator, Type elementType |
     expr.getDeallocator() = deallocator and
-    sig = deallocator.getFullSignature() and
+    sig = getIdentityString(deallocator) and
     elementType = expr.getDeletedElementType() and
     type = elementType.toString() and
     size = elementType.getSize() and

cpp/ql/test/library-tests/noexcept/copy_from_prototype/copy_from_prototype.expected

@@ -1,30 +1,30 @@
-| copy_from_prototype.cpp:3:7:3:7 | a | a<int>::a(a<int> &&) -> void | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
+| copy_from_prototype.cpp:3:7:3:7 | a | void a<int>::a(a<int> const&) | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
-| copy_from_prototype.cpp:3:7:3:7 | a | a<int>::a(const a<int> &) -> void | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
+| copy_from_prototype.cpp:3:7:3:7 | a | void a<int>::a(a<int>&&) | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
-| copy_from_prototype.cpp:3:7:3:7 | operator= | a<int>::operator=(a<int> &&) -> a<int> & | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
+| copy_from_prototype.cpp:3:7:3:7 | operator= | a<int>& a<int>::operator=(a<int> const&) | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
-| copy_from_prototype.cpp:3:7:3:7 | operator= | a<int>::operator=(const a<int> &) -> a<int> & | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
+| copy_from_prototype.cpp:3:7:3:7 | operator= | a<int>& a<int>::operator=(a<int>&&) | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
-| copy_from_prototype.cpp:4:26:4:26 | a | a<<unnamed>>::a<(unnamed template parameter)>() -> void | copy_from_prototype.cpp:3:7:3:7 | a<<unnamed>> | 123 |
+| copy_from_prototype.cpp:4:26:4:26 | a | void a<(unnamed template parameter)>::a<(unnamed template parameter)>() | copy_from_prototype.cpp:3:7:3:7 | a<<unnamed>> | 123 |
-| copy_from_prototype.cpp:4:26:4:26 | a | a<int>::a<(unnamed template parameter)>() -> void | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
+| copy_from_prototype.cpp:4:26:4:26 | a | void a<int>::a<(unnamed template parameter)>() | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
-| copy_from_prototype.cpp:7:7:7:7 | b | b::b() -> void | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
+| copy_from_prototype.cpp:7:7:7:7 | b | void b::b() | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
-| copy_from_prototype.cpp:7:7:7:7 | b | b::b(b &&) -> void | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
+| copy_from_prototype.cpp:7:7:7:7 | b | void b::b(b const&) | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
-| copy_from_prototype.cpp:7:7:7:7 | b | b::b(const b &) -> void | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
+| copy_from_prototype.cpp:7:7:7:7 | b | void b::b(b&&) | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
-| copy_from_prototype.cpp:7:7:7:7 | operator= | b::operator=(b &&) -> b & | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
+| copy_from_prototype.cpp:7:7:7:7 | operator= | b& b::operator=(b const&) | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
-| copy_from_prototype.cpp:7:7:7:7 | operator= | b::operator=(const b &) -> b & | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
+| copy_from_prototype.cpp:7:7:7:7 | operator= | b& b::operator=(b&&) | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
-| copy_from_prototype.cpp:13:7:13:7 | c | c<int>::c(c<int> &&) -> void | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
+| copy_from_prototype.cpp:13:7:13:7 | c | void c<int>::c(c<int> const&) | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
-| copy_from_prototype.cpp:13:7:13:7 | c | c<int>::c(const c<int> &) -> void | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
+| copy_from_prototype.cpp:13:7:13:7 | c | void c<int>::c(c<int>&&) | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
-| copy_from_prototype.cpp:13:7:13:7 | operator= | c<int>::operator=(c<int> &&) -> c<int> & | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
+| copy_from_prototype.cpp:13:7:13:7 | operator= | c<int>& c<int>::operator=(c<int> const&) | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
-| copy_from_prototype.cpp:13:7:13:7 | operator= | c<int>::operator=(const c<int> &) -> c<int> & | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
+| copy_from_prototype.cpp:13:7:13:7 | operator= | c<int>& c<int>::operator=(c<int>&&) | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
-| copy_from_prototype.cpp:14:26:14:26 | c | c<T>::c<(unnamed template parameter)>() -> void | copy_from_prototype.cpp:13:7:13:7 | c<T> | X |
+| copy_from_prototype.cpp:14:26:14:26 | c | void c<T>::c<(unnamed template parameter)>() | copy_from_prototype.cpp:13:7:13:7 | c<T> | X |
-| copy_from_prototype.cpp:14:26:14:26 | c | c<int>::c<(unnamed template parameter)>() -> void | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
+| copy_from_prototype.cpp:14:26:14:26 | c | void c<int>::c<(unnamed template parameter)>() | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
-| copy_from_prototype.cpp:17:7:17:7 | d | d::d() -> void | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
+| copy_from_prototype.cpp:17:7:17:7 | d | void d::d() | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
-| copy_from_prototype.cpp:17:7:17:7 | d | d::d(const d &) -> void | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
+| copy_from_prototype.cpp:17:7:17:7 | d | void d::d(d const&) | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
-| copy_from_prototype.cpp:17:7:17:7 | d | d::d(d &&) -> void | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
+| copy_from_prototype.cpp:17:7:17:7 | d | void d::d(d&&) | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
-| copy_from_prototype.cpp:17:7:17:7 | operator= | d::operator=(const d &) -> d & | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
+| copy_from_prototype.cpp:17:7:17:7 | operator= | d& d::operator=(d const&) | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
-| copy_from_prototype.cpp:17:7:17:7 | operator= | d::operator=(d &&) -> d & | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
+| copy_from_prototype.cpp:17:7:17:7 | operator= | d& d::operator=(d&&) | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
-| copy_from_prototype.cpp:22:8:22:8 | e | e<int>::e(const e<int> &) -> void | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
+| copy_from_prototype.cpp:22:8:22:8 | e | void e<int>::e(e<int> const&) | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
-| copy_from_prototype.cpp:22:8:22:8 | e | e<int>::e(e<int> &&) -> void | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
+| copy_from_prototype.cpp:22:8:22:8 | e | void e<int>::e(e<int>&&) | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
-| copy_from_prototype.cpp:22:8:22:8 | operator= | e<int>::operator=(const e<int> &) -> e<int> & | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
+| copy_from_prototype.cpp:22:8:22:8 | operator= | e<int>& e<int>::operator=(e<int> const&) | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
-| copy_from_prototype.cpp:22:8:22:8 | operator= | e<int>::operator=(e<int> &&) -> e<int> & | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
+| copy_from_prototype.cpp:22:8:22:8 | operator= | e<int>& e<int>::operator=(e<int>&&) | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
-| copy_from_prototype.cpp:23:26:23:26 | e | e<T>::e<(unnamed template parameter)>() -> void | copy_from_prototype.cpp:22:8:22:8 | e<T> | 456 |
+| copy_from_prototype.cpp:23:26:23:26 | e | void e<T>::e<(unnamed template parameter)>() | copy_from_prototype.cpp:22:8:22:8 | e<T> | 456 |
-| copy_from_prototype.cpp:26:35:26:43 | e | e<int>::e<(unnamed template parameter)>() -> void | copy_from_prototype.cpp:22:8:22:8 | e<int> | 456 |
+| copy_from_prototype.cpp:26:35:26:43 | e | void e<int>::e<(unnamed template parameter)>() | copy_from_prototype.cpp:22:8:22:8 | e<int> | 456 |
-| file://:0:0:0:0 | operator= | __va_list_tag::operator=(__va_list_tag &&) -> __va_list_tag & | file://:0:0:0:0 | __va_list_tag | <none> |
+| file://:0:0:0:0 | operator= | __va_list_tag& __va_list_tag::operator=(__va_list_tag const&) | file://:0:0:0:0 | __va_list_tag | <none> |
-| file://:0:0:0:0 | operator= | __va_list_tag::operator=(const __va_list_tag &) -> __va_list_tag & | file://:0:0:0:0 | __va_list_tag | <none> |
+| file://:0:0:0:0 | operator= | __va_list_tag& __va_list_tag::operator=(__va_list_tag&&) | file://:0:0:0:0 | __va_list_tag | <none> |

cpp/ql/test/library-tests/noexcept/copy_from_prototype/copy_from_prototype.ql

@@ -1,4 +1,5 @@
 import cpp
+import semmle.code.cpp.Print
 
 from Function f, string e
 where
@@ -8,4 +9,4 @@ where
     then e = f.getADeclarationEntry().getNoExceptExpr().toString()
     else e = "<no expr>"
   else e = "<none>"
-select f, f.getFullSignature(), f.getDeclaringType(), e
+select f, getIdentityString(f), f.getDeclaringType(), e

cpp/ql/test/query-tests/Critical/MissingCheckScanf/MissingCheckScanf.expected

@@ -19,3 +19,6 @@
 | test.cpp:302:8:302:12 | ptr_i | This variable is read, but may not have been written. It should be guarded by a check that the $@ returns at least 1. | test.cpp:301:3:301:7 | call to scanf | call to scanf |
 | test.cpp:310:7:310:7 | i | This variable is read, but may not have been written. It should be guarded by a check that the $@ returns at least 1. | test.cpp:309:3:309:7 | call to scanf | call to scanf |
 | test.cpp:404:25:404:25 | u | This variable is read, but may not have been written. It should be guarded by a check that the $@ returns at least 1. | test.cpp:403:6:403:11 | call to sscanf | call to sscanf |
+| test.cpp:416:7:416:7 | i | This variable is read, but may not have been written. It should be guarded by a check that the $@ returns at least 1. | test.cpp:413:7:413:11 | call to scanf | call to scanf |
+| test.cpp:423:7:423:7 | i | This variable is read, but may not have been written. It should be guarded by a check that the $@ returns at least 1. | test.cpp:420:7:420:11 | call to scanf | call to scanf |
+| test.cpp:430:6:430:6 | i | This variable is read, but may not have been written. It should be guarded by a check that the $@ returns at least 1. | test.cpp:429:2:429:6 | call to scanf | call to scanf |

cpp/ql/test/query-tests/Critical/MissingCheckScanf/test.cpp

@@ -406,3 +406,26 @@ char *my_string_copy() {
 	*ptr++ = 0;
 	return DST_STRING;
 }
+
+void scan_and_write() {
+	{
+		int i;
+		if (scanf("%d", &i) < 1) {
+			i = 0;
+		}
+		use(i);  // GOOD [FALSE POSITIVE]: variable is overwritten with a default value when scanf fails
+	}
+	{
+		int i;
+		if (scanf("%d", &i) != 1) {
+			i = 0;
+		}
+		use(i);  // GOOD [FALSE POSITIVE]: variable is overwritten with a default value when scanf fails
+	}
+}
+
+void scan_and_static_variable() {
+	static int i;
+	scanf("%d", &i);
+	use(i);  // GOOD [FALSE POSITIVE]: static variables are always 0-initialized
+}

cpp/ql/test/query-tests/Security/CWE/CWE-078/semmle/ExecTainted/ExecTainted.expected

@@ -71,8 +71,6 @@ edges
 | test.cpp:220:10:220:16 | strncat output argument | test.cpp:222:32:222:38 | command indirection |
 | test.cpp:220:19:220:26 | filename indirection | test.cpp:220:10:220:16 | strncat output argument |
 | test.cpp:220:19:220:26 | filename indirection | test.cpp:220:10:220:16 | strncat output argument |
-| test.cpp:220:19:220:26 | filename indirection | test.cpp:220:10:220:16 | strncat output argument |
-| test.cpp:220:19:220:26 | filename indirection | test.cpp:220:10:220:16 | strncat output argument |
 nodes
 | test.cpp:15:27:15:30 | argv | semmle.label | argv |
 | test.cpp:22:13:22:20 | sprintf output argument | semmle.label | sprintf output argument |
@@ -151,6 +149,7 @@ nodes
 | test.cpp:220:19:220:26 | filename indirection | semmle.label | filename indirection |
 | test.cpp:220:19:220:26 | filename indirection | semmle.label | filename indirection |
 | test.cpp:222:32:222:38 | command indirection | semmle.label | command indirection |
+| test.cpp:222:32:222:38 | command indirection | semmle.label | command indirection |
 subpaths
 | test.cpp:196:26:196:33 | filename | test.cpp:186:47:186:54 | filename | test.cpp:188:11:188:17 | command [post update] | test.cpp:196:10:196:16 | command [post update] |
 | test.cpp:196:26:196:33 | filename | test.cpp:186:47:186:54 | filename | test.cpp:188:11:188:17 | command [post update] | test.cpp:196:10:196:16 | command [post update] |

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -1,5 +1,6 @@
 using Xunit;
 using Semmle.Autobuild.Shared;
+using Semmle.Util;
 using System.Collections.Generic;
 using System;
 using System.Linq;
@@ -85,6 +86,15 @@ namespace Semmle.Autobuild.CSharp.Tests
             return ret;
         }
 
+        int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
+        {
+            var ret = (this as IBuildActions).RunProcess(cmd, args, workingDirectory, env, out var stdout);
+
+            stdout.ForEach(line => onOutput(line));
+
+            return ret;
+        }
+
         public IList<string> DirectoryDeleteIn { get; } = new List<string>();
 
         void IBuildActions.DirectoryDelete(string dir, bool recursive)
@@ -200,6 +210,16 @@ namespace Semmle.Autobuild.CSharp.Tests
             if (!DownloadFiles.Contains((address, fileName)))
                 throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");
         }
+
+
+        public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new TestDiagnosticWriter();
+    }
+
+    internal class TestDiagnosticWriter : IDiagnosticsWriter
+    {
+        public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
+
+        public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
     }
 
     /// <summary>
@@ -391,6 +411,7 @@ namespace Semmle.Autobuild.CSharp.Tests
             actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
             actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
+            actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = isWindows ? "win64" : "linux64";
             actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;

csharp/autobuilder/Semmle.Autobuild.CSharp/AutoBuildRule.cs

@@ -0,0 +1,69 @@
+using Semmle.Autobuild.Shared;
+using Semmle.Extraction.CSharp;
+
+namespace Semmle.Autobuild.CSharp
+{
+    internal class AutoBuildRule : IBuildRule<CSharpAutobuildOptions>
+    {
+        private readonly CSharpAutobuilder autobuilder;
+
+        public DotNetRule DotNetRule { get; }
+
+        public MsBuildRule MsBuildRule { get; }
+
+        public BuildCommandAutoRule BuildCommandAutoRule { get; }
+
+        public AutoBuildRule(CSharpAutobuilder autobuilder)
+        {
+            this.autobuilder = autobuilder;
+            this.DotNetRule = new DotNetRule();
+            this.MsBuildRule = new MsBuildRule();
+            this.BuildCommandAutoRule = new BuildCommandAutoRule(DotNetRule.WithDotNet);
+        }
+
+        public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
+        {
+            var cleanTrapFolder =
+                BuildScript.DeleteDirectory(this.autobuilder.TrapDir);
+            var cleanSourceArchive =
+                BuildScript.DeleteDirectory(this.autobuilder.SourceArchiveDir);
+            var tryCleanExtractorArgsLogs =
+                BuildScript.Create(actions =>
+                {
+                    foreach (var file in Extractor.GetCSharpArgsLogs())
+                    {
+                        try
+                        {
+                            actions.FileDelete(file);
+                        }
+                        catch // lgtm[cs/catch-of-all-exceptions] lgtm[cs/empty-catch-block]
+                        { }
+                    }
+
+                    return 0;
+                });
+            var attemptExtractorCleanup =
+                BuildScript.Try(cleanTrapFolder) &
+                BuildScript.Try(cleanSourceArchive) &
+                tryCleanExtractorArgsLogs &
+                BuildScript.DeleteFile(Extractor.GetCSharpLogPath());
+
+            /// <summary>
+            /// Execute script `s` and check that the C# extractor has been executed.
+            /// If either fails, attempt to cleanup any artifacts produced by the extractor,
+            /// and exit with code 1, in order to proceed to the next attempt.
+            /// </summary>
+            BuildScript IntermediateAttempt(BuildScript s) =>
+                (s & this.autobuilder.CheckExtractorRun(false)) |
+                (attemptExtractorCleanup & BuildScript.Failure);
+
+            return
+                // First try .NET Core
+                IntermediateAttempt(this.DotNetRule.Analyse(this.autobuilder, true)) |
+                // Then MSBuild
+                (() => IntermediateAttempt(this.MsBuildRule.Analyse(this.autobuilder, true))) |
+                // And finally look for a script that might be a build script
+                (() => this.BuildCommandAutoRule.Analyse(this.autobuilder, true) & this.autobuilder.CheckExtractorRun(true));
+        }
+    }
+}

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

@@ -1,6 +1,8 @@
 using Semmle.Extraction.CSharp;
 using Semmle.Util.Logging;
 using Semmle.Autobuild.Shared;
+using Semmle.Util;
+using System.Linq;
 
 namespace Semmle.Autobuild.CSharp
 {
@@ -29,25 +31,16 @@ namespace Semmle.Autobuild.CSharp
 
     public class CSharpAutobuilder : Autobuilder<CSharpAutobuildOptions>
     {
-        public CSharpAutobuilder(IBuildActions actions, CSharpAutobuildOptions options) : base(actions, options) { }
+        private const string buildCommandDocsUrl =
+            "https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages";
+
+        private readonly AutoBuildRule autoBuildRule;
+
+        public CSharpAutobuilder(IBuildActions actions, CSharpAutobuildOptions options) : base(actions, options, new CSharpDiagnosticClassifier()) =>
+            this.autoBuildRule = new AutoBuildRule(this);
 
         public override BuildScript GetBuildScript()
         {
-            /// <summary>
-            /// A script that checks that the C# extractor has been executed.
-            /// </summary>
-            BuildScript CheckExtractorRun(bool warnOnFailure) =>
-                BuildScript.Create(actions =>
-                {
-                    if (actions.FileExists(Extractor.GetCSharpLogPath()))
-                        return 0;
-
-                    if (warnOnFailure)
-                        Log(Severity.Error, "No C# code detected during build.");
-
-                    return 1;
-                });
-
             var attempt = BuildScript.Failure;
             switch (GetCSharpBuildStrategy())
             {
@@ -65,47 +58,9 @@ namespace Semmle.Autobuild.CSharp
                     attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true);
                     break;
                 case CSharpBuildStrategy.Auto:
-                    var cleanTrapFolder =
-                        BuildScript.DeleteDirectory(TrapDir);
-                    var cleanSourceArchive =
-                        BuildScript.DeleteDirectory(SourceArchiveDir);
-                    var tryCleanExtractorArgsLogs =
-                        BuildScript.Create(actions =>
-                        {
-                            foreach (var file in Extractor.GetCSharpArgsLogs())
-                            {
-                                try
-                                {
-                                    actions.FileDelete(file);
-                                }
-                                catch // lgtm[cs/catch-of-all-exceptions] lgtm[cs/empty-catch-block]
-                                { }
-                            }
-
-                            return 0;
-                        });
-                    var attemptExtractorCleanup =
-                        BuildScript.Try(cleanTrapFolder) &
-                        BuildScript.Try(cleanSourceArchive) &
-                        tryCleanExtractorArgsLogs &
-                        BuildScript.DeleteFile(Extractor.GetCSharpLogPath());
-
-                    /// <summary>
-                    /// Execute script `s` and check that the C# extractor has been executed.
-                    /// If either fails, attempt to cleanup any artifacts produced by the extractor,
-                    /// and exit with code 1, in order to proceed to the next attempt.
-                    /// </summary>
-                    BuildScript IntermediateAttempt(BuildScript s) =>
-                        (s & CheckExtractorRun(false)) |
-                        (attemptExtractorCleanup & BuildScript.Failure);
-
                     attempt =
-                        // First try .NET Core
+                        // Attempt a few different build strategies to see if one works
-                        IntermediateAttempt(new DotNetRule().Analyse(this, true)) |
+                        this.autoBuildRule.Analyse(this, true) |
-                        // Then MSBuild
-                        (() => IntermediateAttempt(new MsBuildRule().Analyse(this, true))) |
-                        // And finally look for a script that might be a build script
-                        (() => new BuildCommandAutoRule(DotNetRule.WithDotNet).Analyse(this, true) & CheckExtractorRun(true)) |
                         // All attempts failed: print message
                         AutobuildFailure();
                     break;
@@ -114,6 +69,127 @@ namespace Semmle.Autobuild.CSharp
             return attempt;
         }
 
+        /// <summary>
+        /// A script that checks that the C# extractor has been executed.
+        /// </summary>
+        public BuildScript CheckExtractorRun(bool warnOnFailure) =>
+            BuildScript.Create(actions =>
+            {
+                if (actions.FileExists(Extractor.GetCSharpLogPath()))
+                    return 0;
+
+                if (warnOnFailure)
+                    Log(Severity.Error, "No C# code detected during build.");
+
+                return 1;
+            });
+
+        protected override void AutobuildFailureDiagnostic()
+        {
+            // if `ScriptPath` is not null here, the `BuildCommandAuto` rule was
+            // run and found at least one script to execute
+            if (this.autoBuildRule.BuildCommandAutoRule.ScriptPath is not null)
+            {
+                var relScriptPath = this.MakeRelative(autoBuildRule.BuildCommandAutoRule.ScriptPath);
+
+                // if we found multiple build scripts in the project directory, then we can say
+                // as much to indicate that we may have picked the wrong one;
+                // otherwise, we just report that the one script we found didn't work
+                DiagnosticMessage message =
+                    this.autoBuildRule.BuildCommandAutoRule.CandidatePaths.Count() > 1 ?
+                    new(
+                        this.Options.Language,
+                        "multiple-build-scripts",
+                        "There are multiple potential build scripts",
+                        markdownMessage:
+                            "CodeQL found multiple potential build scripts for your project and " +
+                            $"attempted to run `{relScriptPath}`, which failed. " +
+                            "This may not be the right build script for your project. " +
+                            $"Set up a [manual build command]({buildCommandDocsUrl})."
+                    ) :
+                    new(
+                        this.Options.Language,
+                        "script-failure",
+                        "Unable to build project using build script",
+                        markdownMessage:
+                            "CodeQL attempted to build your project using a script located at " +
+                            $"`{relScriptPath}`, which failed. " +
+                            $"Set up a [manual build command]({buildCommandDocsUrl})."
+                    );
+
+                AddDiagnostic(message);
+            }
+
+            // project files which don't exist get marked as not .NET core projects, but we don't want
+            // to show an error for this if the files don't exist
+            var foundNotDotNetProjects = autoBuildRule.DotNetRule.NotDotNetProjects.Where(
+                proj => this.Actions.FileExists(proj.FullPath)
+            );
+
+            // both dotnet and msbuild builds require project or solution files; if we haven't found any
+            // then neither of those rules would've worked
+            if (this.ProjectsOrSolutionsToBuild.Count == 0)
+            {
+                this.AddDiagnostic(new(
+                    this.Options.Language,
+                    "no-projects-or-solutions",
+                    "No project or solutions files found",
+                    markdownMessage:
+                        "CodeQL could not find any project or solution files in your repository. " +
+                        $"Set up a [manual build command]({buildCommandDocsUrl})."
+                ));
+            }
+            // show a warning if there are projects which are not compatible with .NET Core, in case that is unintentional
+            else if (foundNotDotNetProjects.Any())
+            {
+                this.AddDiagnostic(new(
+                    this.Options.Language,
+                    "dotnet-incompatible-projects",
+                    "Some projects are incompatible with .NET Core",
+                    severity: DiagnosticMessage.TspSeverity.Warning,
+                    markdownMessage: $"""
+                    CodeQL found some projects which cannot be built with .NET Core:
+
+                    {autoBuildRule.DotNetRule.NotDotNetProjects.Select(p => this.MakeRelative(p.FullPath)).ToMarkdownList(MarkdownUtil.CodeFormatter, 5)}
+                    """
+                ));
+            }
+
+            // report any projects that failed to build with .NET Core
+            if (autoBuildRule.DotNetRule.FailedProjectsOrSolutions.Any())
+            {
+                this.AddDiagnostic(new(
+                    this.Options.Language,
+                    "dotnet-build-failure",
+                    "Some projects or solutions failed to build using .NET Core",
+                    markdownMessage: $"""
+                    CodeQL was unable to build the following projects using .NET Core:
+
+                    {autoBuildRule.DotNetRule.FailedProjectsOrSolutions.Select(p => this.MakeRelative(p.FullPath)).ToMarkdownList(MarkdownUtil.CodeFormatter, 10)}
+
+                    Set up a [manual build command]({buildCommandDocsUrl}).
+                    """
+                ));
+            }
+
+            // report any projects that failed to build with MSBuild
+            if (autoBuildRule.MsBuildRule.FailedProjectsOrSolutions.Any())
+            {
+                this.AddDiagnostic(new(
+                    this.Options.Language,
+                    "msbuild-build-failure",
+                    "Some projects or solutions failed to build using MSBuild",
+                    markdownMessage: $"""
+                    CodeQL was unable to build the following projects using MSBuild:
+
+                    {autoBuildRule.MsBuildRule.FailedProjectsOrSolutions.Select(p => this.MakeRelative(p.FullPath)).ToMarkdownList(MarkdownUtil.CodeFormatter, 10)}
+
+                    Set up a [manual build command]({buildCommandDocsUrl}).
+                    """
+                ));
+            }
+        }
+
         /// <summary>
         /// Gets the build strategy that the autobuilder should apply, based on the
         /// options in the `lgtm.yml` file.

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpDiagnosticClassifier.cs

@@ -0,0 +1,133 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Semmle.Autobuild.Shared;
+using Semmle.Util;
+
+namespace Semmle.Autobuild.CSharp
+{
+    /// <summary>
+    /// A diagnostic rule which tries to identify missing Xamarin SDKs.
+    /// </summary>
+    public class MissingXamarinSdkRule : DiagnosticRule
+    {
+        private const string docsUrl = "https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-xamarin-applications";
+
+        public class Result : IDiagnosticsResult
+        {
+            /// <summary>
+            /// The name of the SDK that is missing.
+            /// </summary>
+            public string SDKName { get; }
+
+            public Result(string sdkName)
+            {
+                this.SDKName = sdkName;
+            }
+
+            public DiagnosticMessage ToDiagnosticMessage<T>(Autobuilder<T> builder, DiagnosticMessage.TspSeverity? severity = null) where T : AutobuildOptionsShared => new(
+                builder.Options.Language,
+                $"missing-xamarin-{this.SDKName.ToLower()}-sdk",
+                $"Missing Xamarin SDK for {this.SDKName}",
+                severity: severity ?? DiagnosticMessage.TspSeverity.Error,
+                markdownMessage: $"[Configure your workflow]({docsUrl}) for this SDK before running CodeQL."
+            );
+        }
+
+        public MissingXamarinSdkRule() :
+            base("MSB4019:[^\"]*\"[^\"]*Xamarin\\.(?<sdkName>[^\\.]*)\\.CSharp\\.targets\"")
+        {
+        }
+
+        public override void Fire(DiagnosticClassifier classifier, Match match)
+        {
+            if (!match.Groups.TryGetValue("sdkName", out var sdkName))
+                throw new ArgumentException("Expected regular expression match to contain sdkName");
+
+            var xamarinResults = classifier.Results.OfType<Result>().Where(result =>
+                result.SDKName.Equals(sdkName.Value)
+            );
+
+            if (!xamarinResults.Any())
+                classifier.Results.Add(new Result(sdkName.Value));
+        }
+    }
+
+    public class MissingProjectFileRule : DiagnosticRule
+    {
+        private const string runsOnDocsUrl = "https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on";
+        private const string checkoutDocsUrl = "https://github.com/actions/checkout#usage";
+
+        public class Result : IDiagnosticsResult
+        {
+            /// <summary>
+            /// A set of missing project files.
+            /// </summary>
+            public HashSet<string> MissingProjectFiles { get; }
+
+            public Result()
+            {
+                this.MissingProjectFiles = new HashSet<string>();
+            }
+
+            public DiagnosticMessage ToDiagnosticMessage<T>(Autobuilder<T> builder, DiagnosticMessage.TspSeverity? severity = null) where T : AutobuildOptionsShared => new(
+                builder.Options.Language,
+                "missing-project-files",
+                "Missing project files",
+                severity: severity ?? DiagnosticMessage.TspSeverity.Warning,
+                markdownMessage: $"""
+                Some project files were not found when CodeQL built your project:
+
+                {this.MissingProjectFiles.AsEnumerable().Select(p => builder.MakeRelative(p)).ToMarkdownList(MarkdownUtil.CodeFormatter, 5)}
+
+                This may lead to subsequent failures. You can check for common causes for missing project files:
+
+                - Ensure that the project is built using the {runsOnDocsUrl.ToMarkdownLink("intended operating system")} and that filenames on case-sensitive platforms are correctly specified.
+                - If your repository uses Git submodules, ensure that those are {checkoutDocsUrl.ToMarkdownLink("checked out")} before the CodeQL action is run.
+                - If you auto-generate some project files as part of your build process, ensure that these are generated before the CodeQL action is run.
+                """
+            );
+        }
+
+        public MissingProjectFileRule() :
+            base("MSB3202: The project file \"(?<projectFile>[^\"]+)\" was not found. \\[(?<location>[^\\]]+)\\]")
+        {
+        }
+
+        public override void Fire(DiagnosticClassifier classifier, Match match)
+        {
+            if (!match.Groups.TryGetValue("projectFile", out var projectFile))
+                throw new ArgumentException("Expected regular expression match to contain projectFile");
+            if (!match.Groups.TryGetValue("location", out var location))
+                throw new ArgumentException("Expected regular expression match to contain location");
+
+            var result = classifier.Results.OfType<Result>().FirstOrDefault();
+
+            // if we do not yet have a result for this rule, create one and add it to the list
+            // of results the classifier knows about
+            if (result is null)
+            {
+                result = new Result();
+                classifier.Results.Add(result);
+            }
+
+            // then add the missing project file
+            result.MissingProjectFiles.Add(projectFile.Value);
+        }
+    }
+
+    /// <summary>
+    /// Implements a <see cref="DiagnosticClassifier" /> which applies C#-specific rules to
+    /// the build output.
+    /// </summary>
+    public class CSharpDiagnosticClassifier : DiagnosticClassifier
+    {
+        public CSharpDiagnosticClassifier()
+        {
+            // add C#-specific rules to this classifier
+            this.AddRule(new MissingXamarinSdkRule());
+            this.AddRule(new MissingProjectFileRule());
+        }
+    }
+}

csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs

@@ -15,6 +15,15 @@ namespace Semmle.Autobuild.CSharp
     /// </summary>
     internal class DotNetRule : IBuildRule<CSharpAutobuildOptions>
     {
+        public readonly List<IProjectOrSolution> FailedProjectsOrSolutions = new();
+
+        /// <summary>
+        /// A list of projects which are incompatible with DotNet.
+        /// </summary>
+        public IEnumerable<Project<CSharpAutobuildOptions>> NotDotNetProjects { get; private set; }
+
+        public DotNetRule() => NotDotNetProjects = new List<Project<CSharpAutobuildOptions>>();
+
         public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
         {
             if (!builder.ProjectsOrSolutionsToBuild.Any())
@@ -22,10 +31,12 @@ namespace Semmle.Autobuild.CSharp
 
             if (auto)
             {
-                var notDotNetProject = builder.ProjectsOrSolutionsToBuild
+                NotDotNetProjects = builder.ProjectsOrSolutionsToBuild
                     .SelectMany(p => Enumerators.Singleton(p).Concat(p.IncludedProjects))
                     .OfType<Project<CSharpAutobuildOptions>>()
-                    .FirstOrDefault(p => !p.DotNetProject);
+                    .Where(p => !p.DotNetProject);
+                var notDotNetProject = NotDotNetProjects.FirstOrDefault();
+
                 if (notDotNetProject is not null)
                 {
                     builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject);
@@ -50,7 +61,10 @@ namespace Semmle.Autobuild.CSharp
 
                         var build = GetBuildScript(builder, dotNetPath, environment, projectOrSolution.FullPath);
 
-                        ret &= BuildScript.Try(clean) & BuildScript.Try(restore) & build;
+                        ret &= BuildScript.Try(clean) & BuildScript.Try(restore) & BuildScript.OnFailure(build, ret =>
+                        {
+                            FailedProjectsOrSolutions.Add(projectOrSolution);
+                        });
                     }
                     return ret;
                 });

csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
+using Semmle.Util;
 
 namespace Semmle.Autobuild.Shared
 {

csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs

@@ -1,3 +1,4 @@
+using Semmle.Util;
 using Semmle.Util.Logging;
 using System;
 using System.Collections.Generic;
@@ -189,10 +190,11 @@ namespace Semmle.Autobuild.Shared
         /// solution file and tools.
         /// </summary>
         /// <param name="options">The command line options.</param>
-        protected Autobuilder(IBuildActions actions, TAutobuildOptions options)
+        protected Autobuilder(IBuildActions actions, TAutobuildOptions options, DiagnosticClassifier diagnosticClassifier)
         {
             Actions = actions;
             Options = options;
+            DiagnosticClassifier = diagnosticClassifier;
 
             pathsLazy = new Lazy<IEnumerable<(string, int)>>(() =>
             {
@@ -232,24 +234,53 @@ namespace Semmle.Autobuild.Shared
                 return ret ?? new List<IProjectOrSolution>();
             });
 
-            CodeQLExtractorLangRoot = Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_ROOT");
+            CodeQLExtractorLangRoot = Actions.GetEnvironmentVariable(EnvVars.Root(this.Options.Language));
-            CodeQlPlatform = Actions.GetEnvironmentVariable("CODEQL_PLATFORM");
+            CodeQlPlatform = Actions.GetEnvironmentVariable(EnvVars.Platform);
 
-            TrapDir =
+            TrapDir = RequireEnvironmentVariable(EnvVars.TrapDir(this.Options.Language));
-                Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_TRAP_DIR") ??
+            SourceArchiveDir = RequireEnvironmentVariable(EnvVars.SourceArchiveDir(this.Options.Language));
-                throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_TRAP_DIR has not been set.");
+            DiagnosticsDir = RequireEnvironmentVariable(EnvVars.DiagnosticDir(this.Options.Language));
 
-            SourceArchiveDir =
+            this.diagnostics = actions.CreateDiagnosticsWriter(Path.Combine(DiagnosticsDir, $"autobuilder-{DateTime.UtcNow:yyyyMMddHHmm}.jsonc"));
-                Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_SOURCE_ARCHIVE_DIR") ??
-                throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_SOURCE_ARCHIVE_DIR has not been set.");
         }
 
-        protected string TrapDir { get; }
+        /// <summary>
+        /// Retrieves the value of an environment variable named <paramref name="name"> or throws
+        /// an exception if no such environment variable has been set.
+        /// </summary>
+        /// <param name="name">The name of the environment variable.</param>
+        /// <returns>The value of the environment variable.</returns>
+        /// <exception cref="InvalidEnvironmentException">
+        /// Thrown if the environment variable is not set.
+        /// </exception>
+        protected string RequireEnvironmentVariable(string name)
+        {
+            return Actions.GetEnvironmentVariable(name) ??
+                throw new InvalidEnvironmentException($"The environment variable {name} has not been set.");
+        }
 
-        protected string SourceArchiveDir { get; }
+        public string TrapDir { get; }
+
+        public string SourceArchiveDir { get; }
+
+        public string DiagnosticsDir { get; }
+
+        protected DiagnosticClassifier DiagnosticClassifier { get; }
 
         private readonly ILogger logger = new ConsoleLogger(Verbosity.Info);
 
+        private readonly IDiagnosticsWriter diagnostics;
+
+        /// <summary>
+        /// Makes <see cref="path" /> relative to the root source directory.
+        /// </summary>
+        /// <param name="path">The path which to make relative.</param>
+        /// <returns>The relative path.</returns>
+        public string MakeRelative(string path)
+        {
+            return Path.GetRelativePath(this.RootDirectory, path);
+        }
+
         /// <summary>
         /// Log a given build event to the console.
         /// </summary>
@@ -260,6 +291,15 @@ namespace Semmle.Autobuild.Shared
             logger.Log(severity, format, args);
         }
 
+        /// <summary>
+        /// Write <paramref name="diagnostic"/> to the diagnostics file.
+        /// </summary>
+        /// <param name="diagnostic">The diagnostics entry to write.</param>
+        public void AddDiagnostic(DiagnosticMessage diagnostic)
+        {
+            diagnostics.AddEntry(diagnostic);
+        }
+
         /// <summary>
         /// Attempt to build this project.
         /// </summary>
@@ -283,7 +323,19 @@ namespace Semmle.Autobuild.Shared
                 Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
             }
 
-            return script.Run(Actions, startCallback, exitCallback);
+            var onOutput = BuildOutputHandler(Console.Out);
+            var onError = BuildOutputHandler(Console.Error);
+
+            var buildResult = script.Run(Actions, startCallback, exitCallback, onOutput, onError);
+
+            // if the build succeeded, all diagnostics we captured from the build output should be warnings;
+            // otherwise they should all be errors
+            var diagSeverity = buildResult == 0 ? DiagnosticMessage.TspSeverity.Warning : DiagnosticMessage.TspSeverity.Error;
+            this.DiagnosticClassifier.Results
+                .Select(result => result.ToDiagnosticMessage(this, diagSeverity))
+                .ForEach(AddDiagnostic);
+
+            return buildResult;
         }
 
         /// <summary>
@@ -291,13 +343,58 @@ namespace Semmle.Autobuild.Shared
         /// </summary>
         public abstract BuildScript GetBuildScript();
 
+
+        /// <summary>
+        /// Produces a diagnostic for the tool status page that we were unable to automatically
+        /// build the user's project and that they can manually specify a build command. This
+        /// can be overriden to implement more specific messages depending on the origin of
+        /// the failure.
+        /// </summary>
+        protected virtual void AutobuildFailureDiagnostic() => AddDiagnostic(new DiagnosticMessage(
+                this.Options.Language,
+                "autobuild-failure",
+                "Unable to build project",
+                visibility: new DiagnosticMessage.TspVisibility(statusPage: true),
+                plaintextMessage: """
+                    We were unable to automatically build your project.
+                    Set up a manual build command.
+                """
+            ));
+
+        /// <summary>
+        /// Returns a build script that can be run upon autobuild failure.
+        /// </summary>
+        /// <returns>
+        /// A build script that reports that we could not automatically detect a suitable build method.
+        /// </returns>
         protected BuildScript AutobuildFailure() =>
             BuildScript.Create(actions =>
                 {
                     Log(Severity.Error, "Could not auto-detect a suitable build method");
+
+                    AutobuildFailureDiagnostic();
+
                     return 1;
                 });
 
+        /// <summary>
+        /// Constructs a <see cref="BuildOutputHandler" /> which uses the <see cref="DiagnosticClassifier" />
+        /// to classify build output. All data also gets written to <paramref name="writer" />.
+        /// </summary>
+        /// <param name="writer">
+        /// The <see cref="TextWriter" /> to which the build output would have normally been written to.
+        /// This is normally <see cref="Console.Out" /> or <see cref="Console.Error" />.
+        /// </param>
+        /// <returns>The constructed <see cref="BuildOutputHandler" />.</returns>
+        protected BuildOutputHandler BuildOutputHandler(TextWriter writer) => new(data =>
+        {
+            if (data is not null)
+            {
+                writer.WriteLine(data);
+                DiagnosticClassifier.ClassifyLine(data);
+            }
+        });
+
         /// <summary>
         /// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
         /// </summary>

csharp/autobuilder/Semmle.Autobuild.Shared/BuildActions.cs

@@ -11,11 +11,26 @@ using System.Runtime.InteropServices;
 
 namespace Semmle.Autobuild.Shared
 {
+    public delegate void BuildOutputHandler(string? data);
+
     /// <summary>
     /// Wrapper around system calls so that the build scripts can be unit-tested.
     /// </summary>
     public interface IBuildActions
     {
+
+        /// <summary>
+        /// Runs a process, captures its output, and provides it asynchronously.
+        /// </summary>
+        /// <param name="exe">The exe to run.</param>
+        /// <param name="args">The other command line arguments.</param>
+        /// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
+        /// <param name="env">Additional environment variables.</param>
+        /// <param name="onOutput">A handler for stdout output.</param>
+        /// <param name="onError">A handler for stderr output.</param>
+        /// <returns>The process exit code.</returns>
+        int RunProcess(string exe, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError);
+
         /// <summary>
         /// Runs a process and captures its output.
         /// </summary>
@@ -152,6 +167,17 @@ namespace Semmle.Autobuild.Shared
         /// Downloads the resource with the specified URI to a local file.
         /// </summary>
         void DownloadFile(string address, string fileName);
+
+        /// <summary>
+        /// Creates an <see cref="IDiagnosticsWriter" /> for the given <paramref name="filename" />.
+        /// </summary>
+        /// <param name="filename">
+        /// The path suggesting where the diagnostics should be written to.
+        /// </param>
+        /// <returns>
+        /// A <see cref="IDiagnosticsWriter" /> to which diagnostic entries can be added.
+        /// </returns>
+        IDiagnosticsWriter CreateDiagnosticsWriter(string filename);
     }
 
     /// <summary>
@@ -182,6 +208,26 @@ namespace Semmle.Autobuild.Shared
             return pi;
         }
 
+        int IBuildActions.RunProcess(string exe, string args, string? workingDirectory, System.Collections.Generic.IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
+        {
+            var pi = GetProcessStartInfo(exe, args, workingDirectory, env, true);
+            using var p = new Process
+            {
+                StartInfo = pi
+            };
+            p.StartInfo.RedirectStandardError = true;
+            p.OutputDataReceived += new DataReceivedEventHandler((sender, e) => onOutput(e.Data));
+            p.ErrorDataReceived += new DataReceivedEventHandler((sender, e) => onError(e.Data));
+
+            p.Start();
+
+            p.BeginErrorReadLine();
+            p.BeginOutputReadLine();
+
+            p.WaitForExit();
+            return p.ExitCode;
+        }
+
         int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? environment)
         {
             var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment, false);
@@ -253,6 +299,8 @@ namespace Semmle.Autobuild.Shared
         public void DownloadFile(string address, string fileName) =>
             DownloadFileAsync(address, fileName).Wait();
 
+        public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new DiagnosticsStream(filename);
+
         public static IBuildActions Instance { get; } = new SystemBuildActions();
     }
 }

csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandAutoRule.cs

@@ -11,22 +11,42 @@ namespace Semmle.Autobuild.Shared
     {
         private readonly WithDotNet<AutobuildOptionsShared> withDotNet;
 
+        /// <summary>
+        /// A list of paths to files in the project directory which we classified as scripts.
+        /// </summary>
+        public IEnumerable<string> CandidatePaths { get; private set; }
+
+        /// <summary>
+        /// The path of the script we decided to run, if any.
+        /// </summary>
+        public string? ScriptPath { get; private set; }
+
         public BuildCommandAutoRule(WithDotNet<AutobuildOptionsShared> withDotNet)
         {
             this.withDotNet = withDotNet;
+            this.CandidatePaths = new List<string>();
         }
 
+        /// <summary>
+        /// A list of extensions that we consider to be for scripts on Windows.
+        /// </summary>
         private readonly IEnumerable<string> winExtensions = new List<string> {
             ".bat",
             ".cmd",
             ".exe"
         };
 
+        /// <summary>
+        /// A list of extensions that we consider to be for scripts on Linux.
+        /// </summary>
         private readonly IEnumerable<string> linuxExtensions = new List<string> {
             "",
             ".sh"
         };
 
+        /// <summary>
+        /// A list of filenames without extensions that we think might be build scripts.
+        /// </summary>
         private readonly IEnumerable<string> buildScripts = new List<string> {
             "build"
         };
@@ -35,18 +55,25 @@ namespace Semmle.Autobuild.Shared
         {
             builder.Log(Severity.Info, "Attempting to locate build script");
 
+            // a list of extensions for files that we consider to be scripts on the current platform
             var extensions = builder.Actions.IsWindows() ? winExtensions : linuxExtensions;
+            // a list of combined base script names with the current platform's script extensions
+            // e.g. for Linux: build, build.sh
             var scripts = buildScripts.SelectMany(s => extensions.Select(e => s + e));
-            var scriptPath = builder.Paths.Where(p => scripts.Any(p.Item1.ToLower().EndsWith)).OrderBy(p => p.Item2).Select(p => p.Item1).FirstOrDefault();
+            // search through the files in the project directory for paths which end in one of
+            // the names given by `scripts`, then order them by their distance from the root
+            this.CandidatePaths = builder.Paths.Where(p => scripts.Any(p.Item1.ToLower().EndsWith)).OrderBy(p => p.Item2).Select(p => p.Item1);
+            // pick the first matching path, if there is one
+            this.ScriptPath = this.CandidatePaths.FirstOrDefault();
 
-            if (scriptPath is null)
+            if (this.ScriptPath is null)
                 return BuildScript.Failure;
 
             var chmod = new CommandBuilder(builder.Actions);
-            chmod.RunCommand("/bin/chmod", $"u+x {scriptPath}");
+            chmod.RunCommand("/bin/chmod", $"u+x {this.ScriptPath}");
             var chmodScript = builder.Actions.IsWindows() ? BuildScript.Success : BuildScript.Try(chmod.Script);
 
-            var dir = builder.Actions.GetDirectoryName(scriptPath);
+            var dir = builder.Actions.GetDirectoryName(this.ScriptPath);
 
             // A specific .NET Core version may be required
             return chmodScript & withDotNet(builder, environment =>
@@ -58,7 +85,7 @@ namespace Semmle.Autobuild.Shared
                 if (vsTools is not null)
                     command.CallBatFile(vsTools.Path);
 
-                command.RunCommand(scriptPath);
+                command.RunCommand(this.ScriptPath);
                 return command.Script;
             });
         }

csharp/autobuilder/Semmle.Autobuild.Shared/BuildScript.cs

@@ -46,6 +46,33 @@ namespace Semmle.Autobuild.Shared
         /// <returns>The exit code from this build script.</returns>
         public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout);
 
+        /// <summary>
+        /// Runs this build command.
+        /// </summary>
+        /// <param name="actions">
+        /// The interface used to implement the build actions.
+        /// </param>
+        /// <param name="startCallback">
+        /// A call back that is called every time a new process is started. The
+        /// argument to the call back is a textual representation of the process.
+        /// </param>
+        /// <param name="exitCallBack">
+        /// A call back that is called every time a new process exits. The first
+        /// argument to the call back is the exit code, and the second argument is
+        /// an exit message.
+        /// </param>
+        /// <param name="onOutput">
+        /// A handler for data read from stdout.
+        /// </param>
+        /// <param name="onError">
+        /// A handler for data read from stderr.
+        /// </param>
+        /// <returns>The exit code from this build script.</returns>
+        public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError);
+
+        /// <summary>
+        /// A build script which executes an external program or script.
+        /// </summary>
         private class BuildCommand : BuildScript
         {
             private readonly string exe, arguments;
@@ -110,8 +137,29 @@ namespace Semmle.Autobuild.Shared
                 return ret;
             }
 
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError)
+            {
+                startCallback(this.ToString(), silent);
+                var ret = 1;
+                var retMessage = "";
+                try
+                {
+                    ret = actions.RunProcess(exe, arguments, workingDirectory, environment, onOutput, onError);
+                }
+                catch (Exception ex)
+                    when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException)
+                {
+                    retMessage = ex.Message;
+                }
+                exitCallBack(ret, retMessage, silent);
+                return ret;
             }
 
+        }
+
+        /// <summary>
+        /// A build script which runs a C# function.
+        /// </summary>
         private class ReturnBuildCommand : BuildScript
         {
             private readonly Func<IBuildActions, int> func;
@@ -127,8 +175,13 @@ namespace Semmle.Autobuild.Shared
                 stdout = Array.Empty<string>();
                 return func(actions);
             }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError) => func(actions);
         }
 
+        /// <summary>
+        /// Allows two build scripts to be composed sequentially.
+        /// </summary>
         private class BindBuildScript : BuildScript
         {
             private readonly BuildScript s1;
@@ -175,6 +228,32 @@ namespace Semmle.Autobuild.Shared
                 stdout = @out;
                 return ret2;
             }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError)
+            {
+                int ret1;
+                if (s2a is not null)
+                {
+                    var stdout1 = new List<string>();
+                    var onOutputWrapper = new BuildOutputHandler(data =>
+                    {
+                        if (data is not null)
+                            stdout1.Add(data);
+
+                        onOutput(data);
+                    });
+                    ret1 = s1.Run(actions, startCallback, exitCallBack, onOutputWrapper, onError);
+                    return s2a(stdout1, ret1).Run(actions, startCallback, exitCallBack, onOutput, onError);
+                }
+
+                if (s2b is not null)
+                {
+                    ret1 = s1.Run(actions, startCallback, exitCallBack, onOutput, onError);
+                    return s2b(ret1).Run(actions, startCallback, exitCallBack, onOutput, onError);
+                }
+
+                throw new InvalidOperationException("Unexpected error");
+            }
         }
 
         /// <summary>
@@ -260,6 +339,23 @@ namespace Semmle.Autobuild.Shared
         /// </summary>
         public static BuildScript Try(BuildScript s) => s | Success;
 
+        /// <summary>
+        /// Creates a build script that runs the build script <paramref name="s" />. If
+        /// running <paramref name="s" /> fails, <paramref name="k" /> is invoked with
+        /// the exit code.
+        /// </summary>
+        /// <param name="s">The build script to run.</param>
+        /// <param name="k">
+        /// The callback that is invoked if <paramref name="s" /> failed.
+        /// </param>
+        /// <returns>The build script which implements this.</returns>
+        public static BuildScript OnFailure(BuildScript s, Action<int> k) =>
+            new BindBuildScript(s, ret => Create(actions =>
+            {
+                if (!Succeeded(ret)) k(ret);
+                return ret;
+            }));
+
         /// <summary>
         /// Creates a build script that deletes the given directory.
         /// </summary>

csharp/autobuilder/Semmle.Autobuild.Shared/DiagnosticClassifier.cs

@@ -0,0 +1,206 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Semmle.Util;
+
+namespace Semmle.Autobuild.Shared
+{
+    /// <summary>
+    /// Direct results result from the successful application of a <see cref="DiagnosticRule" />,
+    /// which can later be converted to a corresponding <see cref="DiagnosticMessage" />.
+    /// </summary>
+    public interface IDiagnosticsResult
+    {
+        /// <summary>
+        /// Produces a <see cref="DiagnosticMessage" /> corresponding to this result.
+        /// </summary>
+        /// <param name="builder">
+        /// The autobuilder to use for constructing the base <see cref="DiagnosticMessage" />.
+        /// </param>
+        /// <param name="severity">
+        /// An optional severity value which overrides the default severity of the diagnostic.
+        /// </param>
+        /// <returns>The <see cref="DiagnosticMessage" /> corresponding to this result.</returns>
+        DiagnosticMessage ToDiagnosticMessage<T>(Autobuilder<T> builder, DiagnosticMessage.TspSeverity? severity = null) where T : AutobuildOptionsShared;
+    }
+
+    public class DiagnosticRule
+    {
+        /// <summary>
+        /// The pattern against which this rule matches build output.
+        /// </summary>
+        public Regex Pattern { get; }
+
+        /// <summary>
+        /// Constructs a diagnostic rule for the given <paramref name="pattern" />.
+        /// </summary>
+        /// <param name="pattern"></param>
+        public DiagnosticRule(Regex pattern)
+        {
+            this.Pattern = pattern;
+        }
+
+        /// <summary>
+        /// Constructs a diagnostic rule for the given regular expression <paramref name="pattern" />.
+        /// </summary>
+        /// <param name="pattern"></param>
+        public DiagnosticRule(string pattern)
+        {
+            this.Pattern = new Regex(pattern, RegexOptions.Compiled);
+        }
+
+        /// <summary>
+        /// Used by a <see cref="DiagnosticClassifier" /> <paramref name="classifier" /> to
+        /// signal that the rule has matched some build output with <paramref name="match" />.
+        /// </summary>
+        /// <param name="classifier">The classifier which is firing the rule.</param>
+        /// <param name="match">The <see cref="Match" /> that resulted from applying the rule.</param>
+        public virtual void Fire(DiagnosticClassifier classifier, Match match) { }
+    }
+
+    /// <summary>
+    /// A <see cref="DiagnosticRule" /> which detects arbitrary build messages from MSBuild, CSC, NETSDK, etc. and
+    /// reports them, up to a limit, as telemetry-only diagnostics.
+    /// </summary>
+    public class BuildMessageRule : DiagnosticRule
+    {
+        public class Result : IDiagnosticsResult, IEquatable<Result>
+        {
+            /// <summary>
+            /// A value indicating whether this a warning or an error.
+            /// </summary>
+            public string Type { get; }
+            /// <summary>
+            /// The source of the message, such as "MSB" or "NETSDK".
+            /// </summary>
+            public string Source { get; }
+            /// <summary>
+            /// The numeric id of the message.
+            /// </summary>
+            public int Code { get; }
+            /// <summary>
+            /// The message contents.
+            /// </summary>
+            public string Message { get; }
+
+            public Result(string type, string source, int code, string message)
+            {
+                this.Type = type;
+                this.Source = source;
+                this.Code = code;
+                this.Message = message;
+            }
+
+            public DiagnosticMessage ToDiagnosticMessage<T>(Autobuilder<T> builder, DiagnosticMessage.TspSeverity? severity = null) where T : AutobuildOptionsShared => new(
+                builder.Options.Language,
+                $"{this.Source.ToLower()}-{this.Code}",
+                $"{this.Source.ToUpper()}{this.Code}",
+                plaintextMessage: this.Message,
+                severity:
+                    this.Type.Equals("error") ?
+                    DiagnosticMessage.TspSeverity.Error :
+                    DiagnosticMessage.TspSeverity.Warning,
+                // the messages we capture here are visible in the build log, so there is no need
+                // to show them other than in telemetry
+                visibility: new(telemetry: true)
+            );
+
+            public bool Equals(Result? x)
+            {
+                return x is not null &&
+                    x.Code == this.Code &&
+                    x.Type == this.Type &&
+                    x.Source == this.Source &&
+                    x.Message == this.Message;
+            }
+
+            public override bool Equals(object? obj)
+            {
+                return obj is Result && this.Equals(obj);
+            }
+
+            public override int GetHashCode()
+            {
+                return HashCode.Combine(this.Type, this.Source, this.Code, this.Message);
+            }
+        }
+
+        /// <summary>
+        /// The maximum number of diagnostics we should emit for this rule.
+        /// </summary>
+        private const int maxDiagnostics = 10;
+        /// <summary>
+        /// The number of diagnostics this rule has emitted so far.
+        /// </summary>
+        private int diagnostics = 0;
+
+        public BuildMessageRule() : base("(?<type>error|warning) (?<source>[A-Z]+)(?<code>\\d+): (?<message>.*)") { }
+
+        public override void Fire(DiagnosticClassifier classifier, Match match)
+        {
+            if (!match.Groups.TryGetValue("type", out var type))
+                throw new ArgumentException("Expected regular expression match to contain type");
+            if (!match.Groups.TryGetValue("source", out var source))
+                throw new ArgumentException("Expected regular expression match to contain source");
+            if (!match.Groups.TryGetValue("code", out var codeStr))
+                throw new ArgumentException("Expected regular expression match to contain code");
+            if (!match.Groups.TryGetValue("message", out var message))
+                throw new ArgumentException("Expected regular expression match to contain message");
+            if (!int.TryParse(codeStr.Value, out var code))
+                throw new ArgumentException("Expected code to be numeric");
+
+            // check that we have not yet exceeded our limit for emitting diagnostics
+            if (this.diagnostics < BuildMessageRule.maxDiagnostics)
+            {
+                var result = new Result(type.Value, source.Value, code, message.Value);
+
+                // add this result if we don't already have an identical one
+                if (!classifier.Results.OfType<Result>().Any(d => d.Equals(result)))
+                {
+                    classifier.Results.Add(result);
+                    this.diagnostics++;
+                }
+            }
+        }
+    }
+
+    public class DiagnosticClassifier
+    {
+        private readonly List<DiagnosticRule> rules;
+        public readonly List<IDiagnosticsResult> Results;
+
+        public DiagnosticClassifier()
+        {
+            this.rules = new List<DiagnosticRule>();
+            this.Results = new List<IDiagnosticsResult>();
+
+            this.AddRule(new BuildMessageRule());
+        }
+
+        /// <summary>
+        /// Adds <paramref name="rule" /> to this classifier.
+        /// </summary>
+        /// <param name="rule">The rule to add.</param>
+        protected void AddRule(DiagnosticRule rule)
+        {
+            this.rules.Add(rule);
+        }
+
+        /// <summary>
+        /// Applies all of this classifier's rules to <paramref name="line" /> to see which match.
+        /// </summary>
+        /// <param name="line">The line to which the rules should be applied to.</param>
+        public void ClassifyLine(string line)
+        {
+            this.rules.ForEach(rule =>
+            {
+                var match = rule.Pattern.Match(line);
+                if (match.Success)
+                {
+                    rule.Fire(this, match);
+                }
+            });
+        }
+    }
+}

csharp/autobuilder/Semmle.Autobuild.Shared/EnvVars.cs

@@ -0,0 +1,13 @@
+using Semmle.Util;
+
+namespace Semmle.Autobuild.Shared
+{
+    public static class EnvVars
+    {
+        public const string Platform = "CODEQL_PLATFORM";
+        public static string Root(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_ROOT";
+        public static string TrapDir(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_TRAP_DIR";
+        public static string SourceArchiveDir(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_SOURCE_ARCHIVE_DIR";
+        public static string DiagnosticDir(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_DIAGNOSTIC_DIR";
+    }
+}

csharp/autobuilder/Semmle.Autobuild.Shared/MarkdownUtil.cs

@@ -0,0 +1,65 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Semmle.Autobuild.Shared
+{
+    public static class MarkdownUtil
+    {
+        /// <summary>
+        /// Formats items as markdown inline code.
+        /// </summary>
+        /// <returns>A function which formats items as markdown inline code.</returns>
+        public static readonly Func<string, string> CodeFormatter = item => $"`{item}`";
+
+        /// <summary>
+        /// Formats the string as a markdown link.
+        /// </summary>
+        /// <param name="link">The URL for the link.</param>
+        /// <param name="title">The text that is displayed.</param>
+        /// <returns>A string containing a markdown-formatted link.</returns>
+        public static string ToMarkdownLink(this string link, string title) => $"[{title}]({link})";
+
+        /// <summary>
+        /// Renders <see cref="projects" /> as a markdown list of the project paths.
+        /// </summary>
+        /// <param name="projects">
+        /// The list of projects whose paths should be rendered as a markdown list.
+        /// </param>
+        /// <param name="limit">The maximum number of items to include in the list.</param>
+        /// <returns>Returns the markdown list as a string.</returns>
+        public static string ToMarkdownList(this IEnumerable<IProjectOrSolution> projects, int? limit = null)
+        {
+            return projects.ToMarkdownList(p => $"`{p.FullPath}`", limit);
+        }
+
+        /// <summary>
+        /// Renders <see cref="items" /> as a markdown list.
+        /// </summary>
+        /// <typeparam name="T">The item type.</typeparam>
+        /// <param name="items">The list that should be formatted as a markdown list.</param>
+        /// <param name="formatter">A function which converts individual items into a string representation.</param>
+        /// <param name="limit">The maximum number of items to include in the list.</param>
+        /// <returns>Returns the markdown list as a string.</returns>
+        public static string ToMarkdownList<T>(this IEnumerable<T> items, Func<T, string> formatter, int? limit = null)
+        {
+            var sb = new StringBuilder();
+
+            // if there is a limit, take at most that many items from the  start of the list
+            var list = limit is not null ? items.Take(limit.Value) : items;
+            sb.Append(string.Join('\n', list.Select(item => $"- {formatter(item)}")));
+
+            // if there were more items than allowed in the list, add an extra item indicating
+            // how many more items there were
+            var length = items.Count();
+
+            if (limit is not null && length > limit)
+            {
+                sb.Append($"\n- and {length - limit} more. View the CodeQL logs for a full list.");
+            }
+
+            return sb.ToString();
+        }
+    }
+}

csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs

@@ -1,7 +1,6 @@
 using Semmle.Util.Logging;
-using System;
+using System.Collections.Generic;
 using System.Linq;
-using System.Runtime.InteropServices;
 
 namespace Semmle.Autobuild.Shared
 {
@@ -31,6 +30,11 @@ namespace Semmle.Autobuild.Shared
     /// </summary>
     public class MsBuildRule : IBuildRule<AutobuildOptionsShared>
     {
+        /// <summary>
+        /// A list of solutions or projects which failed to build.
+        /// </summary>
+        public readonly List<IProjectOrSolution> FailedProjectsOrSolutions = new();
+
         public BuildScript Analyse(IAutobuilder<AutobuildOptionsShared> builder, bool auto)
         {
             if (!builder.ProjectsOrSolutionsToBuild.Any())
@@ -128,7 +132,13 @@ namespace Semmle.Autobuild.Shared
 
                 command.Argument(builder.Options.MsBuildArguments);
 
-                ret &= command.Script;
+                // append the build script which invokes msbuild to the overall build script `ret`;
+                // we insert a check that building the current project or solution was successful:
+                // if it was not successful, we add it to `FailedProjectsOrSolutions`
+                ret &= BuildScript.OnFailure(command.Script, ret =>
+                {
+                    FailedProjectsOrSolutions.Add(projectOrSolution);
+                });
             }
 
             return ret;

csharp/autobuilder/Semmle.Autobuild.Shared/ProjectOrSolution.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using Semmle.Util;
 
 namespace Semmle.Autobuild.Shared
 {

csharp/extractor/Semmle.Util/Language.cs

@@ -1,4 +1,4 @@
-namespace Semmle.Autobuild.Shared
+namespace Semmle.Util
 {
     public sealed class Language
     {

csharp/extractor/Semmle.Util/Semmle.Util.csproj

@@ -15,6 +15,7 @@
 
   <ItemGroup>
     <PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
   </ItemGroup>
 
 </Project>

csharp/extractor/Semmle.Util/ToolStatusPage.cs

@@ -0,0 +1,242 @@
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+using Newtonsoft.Json.Serialization;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// Represents diagnostic messages for the tool status page.
+    /// </summary>
+    public class DiagnosticMessage
+    {
+        /// <summary>
+        /// Represents sources of diagnostic messages.
+        /// </summary>
+        public class TspSource
+        {
+            /// <summary>
+            /// An identifier under which it makes sense to group this diagnostic message.
+            /// This is used to build the SARIF reporting descriptor object.
+            /// </summary>
+            public string Id { get; }
+            /// <summary>
+            /// Display name for the ID. This is used to build the SARIF reporting descriptor object.
+            /// </summary>
+            public string Name { get; }
+            /// <summary>
+            /// Name of the CodeQL extractor. This is used to identify which tool component the reporting descriptor object should be nested under in SARIF.
+            /// </summary>
+            public string? ExtractorName { get; }
+
+            public TspSource(string id, string name, string? extractorName = null)
+            {
+                Id = id;
+                Name = name;
+                ExtractorName = extractorName;
+            }
+        }
+
+        /// <summary>
+        /// Enumerates severity levels for diagnostics.
+        /// </summary>
+        [JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]
+        public enum TspSeverity
+        {
+            Note,
+            Warning,
+            Error
+        }
+
+        /// <summary>
+        /// Stores flags indicating where the diagnostic should be displayed.
+        /// </summary>
+        public class TspVisibility
+        {
+            /// <summary>
+            /// A read-only instance of <see cref="TspVisibility" /> which indicates that the
+            /// diagnostic should be used in all supported locations.
+            /// </summary>
+            public static readonly TspVisibility All = new(true, true, true);
+
+            /// <summary>
+            /// True if the message should be displayed on the status page (defaults to false).
+            /// </summary>
+            public bool? StatusPage { get; }
+            /// <summary>
+            /// True if the message should be counted in the diagnostics summary table printed by
+            /// <c>codeql database analyze</c> (defaults to false).
+            /// </summary>
+            public bool? CLISummaryTable { get; }
+            /// <summary>
+            /// True if the message should be sent to telemetry (defaults to false).
+            /// </summary>
+            public bool? Telemetry { get; }
+
+            public TspVisibility(bool? statusPage = null, bool? cliSummaryTable = null, bool? telemetry = null)
+            {
+                this.StatusPage = statusPage;
+                this.CLISummaryTable = cliSummaryTable;
+                this.Telemetry = telemetry;
+            }
+        }
+
+        /// <summary>
+        /// Represents source code locations for diagnostic messages.
+        /// </summary>
+        public class TspLocation
+        {
+            /// <summary>
+            /// Path to the affected file if appropriate, relative to the source root.
+            /// </summary>
+            public string? File { get; }
+            /// <summary>
+            /// The line where the range to which the diagnostic relates to starts.
+            /// </summary>
+            public int? StartLine { get; }
+            /// <summary>
+            /// The column where the range to which the diagnostic relates to starts.
+            /// </summary>
+            public int? StartColumn { get; }
+            /// <summary>
+            /// The line where the range to which the diagnostic relates to ends.
+            /// </summary>
+            public int? EndLine { get; }
+            /// <summary>
+            /// The column where the range to which the diagnostic relates to ends.
+            /// </summary>
+            public int? EndColumn { get; }
+
+            public TspLocation(string? file = null, int? startLine = null, int? startColumn = null, int? endLine = null, int? endColumn = null)
+            {
+                this.File = file;
+                this.StartLine = startLine;
+                this.StartColumn = startColumn;
+                this.EndLine = endLine;
+                this.EndColumn = endColumn;
+            }
+        }
+
+        /// <summary>
+        /// ISO 8601 timestamp.
+        /// </summary>
+        public string Timestamp { get; }
+        /// <summary>
+        /// The source of the diagnostic message.
+        /// </summary>
+        public TspSource Source { get; }
+        /// <summary>
+        /// GitHub flavored Markdown formatted message. Should include inline links to any help pages.
+        /// </summary>
+        public string? MarkdownMessage { get; }
+        /// <summary>
+        /// Plain text message. Used by components where the string processing needed to support
+        /// Markdown is cumbersome.
+        /// </summary>
+        public string? PlaintextMessage { get; }
+        /// <summary>
+        /// List of help links intended to supplement <see cref="PlaintextMessage" />.
+        /// </summary>
+        public List<string> HelpLinks { get; }
+        /// <summary>
+        /// SARIF severity.
+        /// </summary>
+        public TspSeverity? Severity { get; }
+        /// <summary>
+        /// If true, then this message won't be presented to users.
+        /// </summary>
+        public bool Internal { get; }
+        public TspVisibility Visibility { get; }
+        public TspLocation Location { get; }
+        /// <summary>
+        /// Structured metadata about the diagnostic message.
+        /// </summary>
+        public Dictionary<string, object> Attributes { get; }
+
+        public DiagnosticMessage(
+            Language language, string id, string name, string? markdownMessage = null, string? plaintextMessage = null,
+            TspVisibility? visibility = null, TspLocation? location = null, TspSeverity? severity = TspSeverity.Error,
+            DateTime? timestamp = null, bool? intrnl = null
+        )
+        {
+            this.Source = new TspSource(
+                id: $"{language.UpperCaseName.ToLower()}/autobuilder/{id}",
+                name: name,
+                extractorName: language.UpperCaseName.ToLower()
+            );
+            this.Timestamp = (timestamp ?? DateTime.UtcNow).ToString("o", CultureInfo.InvariantCulture);
+            this.HelpLinks = new List<string>();
+            this.Attributes = new Dictionary<string, object>();
+            this.Severity = severity;
+            this.Visibility = visibility ?? TspVisibility.All;
+            this.Location = location ?? new TspLocation();
+            this.Internal = intrnl ?? false;
+            this.MarkdownMessage = markdownMessage;
+            this.PlaintextMessage = plaintextMessage;
+        }
+    }
+
+    /// <summary>
+    /// Provides the ability to write diagnostic messages to some output.
+    /// </summary>
+    public interface IDiagnosticsWriter
+    {
+        /// <summary>
+        /// Adds <paramref name="message" /> as a new diagnostics entry.
+        /// </summary>
+        /// <param name="message">The diagnostics entry to add.</param>
+        void AddEntry(DiagnosticMessage message);
+    }
+
+    /// <summary>
+    /// A wrapper around an underlying <see cref="StreamWriter" /> which allows
+    /// <see cref="DiagnosticMessage" /> objects to be serialized to it.
+    /// </summary>
+    public sealed class DiagnosticsStream : IDiagnosticsWriter, IDisposable
+    {
+        private readonly JsonSerializer serializer;
+        private readonly StreamWriter writer;
+
+        /// <summary>
+        /// Initialises a new <see cref="DiagnosticsStream" /> for a file at <paramref name="path" />.
+        /// </summary>
+        /// <param name="path">The path to the file that should be created.</param>
+        public DiagnosticsStream(string path)
+        {
+            this.writer = File.CreateText(path);
+
+            var contractResolver = new DefaultContractResolver
+            {
+                NamingStrategy = new CamelCaseNamingStrategy()
+            };
+
+            serializer = new JsonSerializer
+            {
+                ContractResolver = contractResolver,
+                NullValueHandling = NullValueHandling.Ignore
+            };
+        }
+
+        /// <summary>
+        /// Adds <paramref name="message" /> as a new diagnostics entry.
+        /// </summary>
+        /// <param name="message">The diagnostics entry to add.</param>
+        public void AddEntry(DiagnosticMessage message)
+        {
+            serializer.Serialize(writer, message);
+            writer.Flush();
+        }
+
+        /// <summary>
+        /// Releases all resources used by the <see cref="DiagnosticsStream" /> object.
+        /// </summary>
+        public void Dispose()
+        {
+            writer.Dispose();
+        }
+    }
+}

csharp/ql/integration-tests/all-platforms/diag_dotnet_incompatible/diagnostics.expected

@@ -0,0 +1,68 @@
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL found some projects which cannot be built with .NET Core:\n\n- `test.csproj`",
+  "severity": "warning",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/dotnet-incompatible-projects",
+    "name": "Some projects are incompatible with .NET Core"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL was unable to build the following projects using MSBuild:\n\n- `test.csproj`\n\nSet up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/msbuild-build-failure",
+    "name": "Some projects or solutions failed to build using MSBuild"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "plaintextMessage": "The reference assemblies for .NETFramework,Version=v4.0 were not found. To resolve this, install the Developer Pack (SDK/Targeting Pack) for this framework version or retarget your application. You can download .NET Framework Developer Packs at https://aka.ms/msbuild/developerpacks [<test-root-directory>/test.csproj]",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/msb-3644",
+    "name": "MSB3644"
+  },
+  "visibility": {
+    "telemetry": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "plaintextMessage": "Too many project files specified",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/msbuild-4",
+    "name": "MSBUILD4"
+  },
+  "visibility": {
+    "telemetry": true
+  }
+}

csharp/ql/integration-tests/all-platforms/diag_dotnet_incompatible/test.csproj

@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>Example</RootNamespace>
+    <AssemblyName>Example</AssemblyName>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <TargetFrameworkProfile />
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x86'">
+    <DebugSymbols>true</DebugSymbols>
+    <OutputPath>bin\x86\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <DebugType>full</DebugType>
+    <PlatformTarget>x86</PlatformTarget>
+    <ErrorReport>prompt</ErrorReport>
+    <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x86'">
+    <OutputPath>bin\x86\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <Optimize>true</Optimize>
+    <DebugType>pdbonly</DebugType>
+    <PlatformTarget>x86</PlatformTarget>
+    <ErrorReport>prompt</ErrorReport>
+    <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Program.cs" />
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>

csharp/ql/integration-tests/all-platforms/diag_dotnet_incompatible/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
+check_diagnostics()

csharp/ql/integration-tests/all-platforms/diag_missing_project_files/diagnostics.expected

@@ -0,0 +1,68 @@
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL was unable to build the following projects using MSBuild:\n\n- `test.sln`\n\nSet up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/msbuild-build-failure",
+    "name": "Some projects or solutions failed to build using MSBuild"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "Some project files were not found when CodeQL built your project:\n\n- `Example.csproj`\n- `Example.Test.csproj`\n\nThis may lead to subsequent failures. You can check for common causes for missing project files:\n\n- Ensure that the project is built using the [intended operating system](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on) and that filenames on case-sensitive platforms are correctly specified.\n- If your repository uses Git submodules, ensure that those are [checked out](https://github.com/actions/checkout#usage) before the CodeQL action is run.\n- If you auto-generate some project files as part of your build process, ensure that these are generated before the CodeQL action is run.",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/missing-project-files",
+    "name": "Missing project files"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "plaintextMessage": "The project file \"<test-root-directory>/Example.Test.csproj\" was not found. [<test-root-directory>/test.sln]",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/msb-3202",
+    "name": "MSB3202"
+  },
+  "visibility": {
+    "telemetry": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "plaintextMessage": "The project file \"<test-root-directory>/Example.csproj\" was not found. [<test-root-directory>/test.sln]",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/msb-3202",
+    "name": "MSB3202"
+  },
+  "visibility": {
+    "telemetry": true
+  }
+}

csharp/ql/integration-tests/all-platforms/diag_missing_project_files/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
+check_diagnostics()

csharp/ql/integration-tests/all-platforms/diag_missing_project_files/test.sln

@@ -0,0 +1,26 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2012
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example", "Example.csproj", "{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example.Test", "Example.Test.csproj", "{F4587B5F-9918-4079-9291-5A08CD9761FB}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x86 = Debug|x86
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}.Debug|x86.ActiveCfg = Debug|x86
+		{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}.Debug|x86.Build.0 = Debug|x86
+		{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}.Release|x86.ActiveCfg = Release|x86
+		{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}.Release|x86.Build.0 = Release|x86
+		{F4587B5F-9918-4079-9291-5A08CD9761FB}.Debug|x86.ActiveCfg = Debug|x86
+		{F4587B5F-9918-4079-9291-5A08CD9761FB}.Debug|x86.Build.0 = Debug|x86
+		{F4587B5F-9918-4079-9291-5A08CD9761FB}.Release|x86.ActiveCfg = Release|x86
+		{F4587B5F-9918-4079-9291-5A08CD9761FB}.Release|x86.Build.0 = Release|x86
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

csharp/ql/integration-tests/all-platforms/diag_missing_xamarin_sdk/diagnostics.expected

@@ -0,0 +1,86 @@
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL was unable to build the following projects using .NET Core:\n\n- `test.csproj`\n\nSet up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/dotnet-build-failure",
+    "name": "Some projects or solutions failed to build using .NET Core"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL was unable to build the following projects using MSBuild:\n\n- `test.csproj`\n\nSet up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/msbuild-build-failure",
+    "name": "Some projects or solutions failed to build using MSBuild"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "[Configure your workflow](https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-xamarin-applications) for this SDK before running CodeQL.",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/missing-xamarin-ios-sdk",
+    "name": "Missing Xamarin SDK for iOS"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "plaintextMessage": "The imported project \"/usr/local/share/dotnet/sdk/7.0.102/Xamarin/iOS/Xamarin.iOS.CSharp.targets\" was not found. Confirm that the expression in the Import declaration \"/usr/local/share/dotnet/sdk/7.0.102/Xamarin/iOS/Xamarin.iOS.CSharp.targets\" is correct, and that the file exists on disk. [<test-root-directory>/test.csproj]",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/msb-4019",
+    "name": "MSB4019"
+  },
+  "visibility": {
+    "telemetry": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "plaintextMessage": "Too many project files specified",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/msbuild-4",
+    "name": "MSBUILD4"
+  },
+  "visibility": {
+    "telemetry": true
+  }
+}

csharp/ql/integration-tests/all-platforms/diag_missing_xamarin_sdk/test.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <LanguageTargets>$(MSBuildExtensionsPath)\Xamarin\iOS\Xamarin.iOS.CSharp.targets</LanguageTargets>
+  </PropertyGroup>
+
+</Project>

csharp/ql/integration-tests/all-platforms/diag_missing_xamarin_sdk/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
+check_diagnostics()

csharp/ql/integration-tests/all-platforms/dotnet_build/test.py

@@ -1,3 +1,5 @@
 from create_database_utils import *
+from diagnostics_test_utils import *
 
-run_codeql_database_create(['dotnet build'], test_db="default-db", db=None, lang="csharp")
+run_codeql_database_create(['dotnet build'], db=None, lang="csharp")
+check_diagnostics()

csharp/ql/integration-tests/all-platforms/dotnet_pack/test.py

@@ -1,8 +1,11 @@
 import os
 from create_database_utils import *
+from diagnostics_test_utils import *
 
-run_codeql_database_create(['dotnet pack'], test_db="default-db", db=None, lang="csharp")
+run_codeql_database_create(['dotnet pack'], db=None, lang="csharp")
 
 ## Check that the NuGet package is created.
 if not os.path.isfile("bin/Debug/dotnet_pack.1.0.0.nupkg"):
     raise Exception("The NuGet package was not created.")
+
+check_diagnostics()

csharp/ql/integration-tests/all-platforms/dotnet_publish/test.py

@@ -1,9 +1,12 @@
 import os
 from create_database_utils import *
+from diagnostics_test_utils import *
 
 artifacts = 'bin/Temp'
-run_codeql_database_create([f"dotnet publish -o {artifacts}"], test_db="default-db", db=None, lang="csharp")
+run_codeql_database_create([f"dotnet publish -o {artifacts}"], db=None, lang="csharp")
 
 ## Check that the publish folder is created.
 if not os.path.isdir(artifacts):
     raise Exception("The publish artifact folder was not created.")
+
+check_diagnostics()

csharp/ql/integration-tests/all-platforms/dotnet_run/test.py

@@ -1,5 +1,6 @@
 import os
 from create_database_utils import *
+from diagnostics_test_utils import *
 
 def run_codeql_database_create_stdout(args, dbname):
     stdout = open(dbname + "file.txt", 'w+')
@@ -16,32 +17,40 @@ def check_build_out(msg, s):
 # no arguments
 s = run_codeql_database_create_stdout(['dotnet run'], "test-db")
 check_build_out("Default reply", s)
+check_diagnostics()
 
 # no arguments, but `--`
 s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test-db', 'dotnet run --'], "test2-db")
 check_build_out("Default reply", s)
+check_diagnostics(diagnostics_dir="test2-db/diagnostic")
 
 # one argument, no `--`
 s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test2-db', 'dotnet run hello'], "test3-db")
 check_build_out("Default reply", s)
+check_diagnostics(diagnostics_dir="test3-db/diagnostic")
 
 # one argument, but `--`
 s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test3-db', 'dotnet run -- hello'], "test4-db")
 check_build_out("Default reply", s)
+check_diagnostics(diagnostics_dir="test4-db/diagnostic")
 
 # two arguments, no `--`
 s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test4-db', 'dotnet run hello world'], "test5-db")
 check_build_out("hello, world", s)
+check_diagnostics(diagnostics_dir="test5-db/diagnostic")
 
 # two arguments, and `--`
 s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test5-db', 'dotnet run -- hello world'], "test6-db")
 check_build_out("hello, world", s)
+check_diagnostics(diagnostics_dir="test6-db/diagnostic")
 
 # shared compilation enabled; tracer should override by changing the command
 # to `dotnet run -p:UseSharedCompilation=true -p:UseSharedCompilation=false -- hello world`
 s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test6-db', 'dotnet run -p:UseSharedCompilation=true -- hello world'], "test7-db")
 check_build_out("hello, world", s)
+check_diagnostics(diagnostics_dir="test7-db/diagnostic")
 
 # option passed into `dotnet run`
 s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test7-db', 'dotnet build', 'dotnet run --no-build hello world'], "test8-db")
 check_build_out("hello, world", s)
+check_diagnostics(diagnostics_dir="test8-db/diagnostic")

csharp/ql/integration-tests/all-platforms/msbuild/diagnostics.expected

@@ -0,0 +1,16 @@
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "plaintextMessage": "Too many project files specified",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/msbuild-4",
+    "name": "MSBUILD4"
+  },
+  "visibility": {
+    "telemetry": true
+  }
+}

csharp/ql/integration-tests/all-platforms/msbuild/test.py

@@ -1,4 +1,6 @@
 from create_database_utils import *
+from diagnostics_test_utils import *
 
 # force CodeQL to use MSBuild by setting `LGTM_INDEX_MSBUILD_TARGET`
-run_codeql_database_create([], test_db="default-db", db=None, lang="csharp", extra_env={ 'LGTM_INDEX_MSBUILD_TARGET': 'Build' })
+run_codeql_database_create([], db=None, lang="csharp", extra_env={ 'LGTM_INDEX_MSBUILD_TARGET': 'Build' })
+check_diagnostics()

csharp/ql/integration-tests/posix-only/diag_autobuild_script/build.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exit 1

csharp/ql/integration-tests/posix-only/diag_autobuild_script/diagnostics.expected

@@ -0,0 +1,36 @@
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL attempted to build your project using a script located at `build.sh`, which failed. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/script-failure",
+    "name": "Unable to build project using build script"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL could not find any project or solution files in your repository. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/no-projects-or-solutions",
+    "name": "No project or solutions files found"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

csharp/ql/integration-tests/posix-only/diag_autobuild_script/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
+check_diagnostics()

csharp/ql/integration-tests/posix-only/diag_multiple_scripts/build.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exit 1

csharp/ql/integration-tests/posix-only/diag_multiple_scripts/diagnostics.expected

@@ -0,0 +1,36 @@
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL could not find any project or solution files in your repository. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/no-projects-or-solutions",
+    "name": "No project or solutions files found"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL found multiple potential build scripts for your project and attempted to run `build.sh`, which failed. This may not be the right build script for your project. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/multiple-build-scripts",
+    "name": "There are multiple potential build scripts"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

csharp/ql/integration-tests/posix-only/diag_multiple_scripts/scripts/build.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exit 1

csharp/ql/integration-tests/posix-only/diag_multiple_scripts/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
+check_diagnostics()

csharp/ql/integration-tests/posix-only/dotnet_test/test.py

@@ -1,3 +1,5 @@
 from create_database_utils import *
+from diagnostics_test_utils import *
 
-run_codeql_database_create(['dotnet test'], test_db="default-db", db=None, lang="csharp")
+run_codeql_database_create(['dotnet test'], db=None, lang="csharp")
+check_diagnostics()

csharp/ql/integration-tests/posix-only/inherit-env-vars/test.py

@@ -1,6 +1,8 @@
 from create_database_utils import *
+from diagnostics_test_utils import *
 import os
 
 os.environ["PROJECT_TO_BUILD"] = "proj.csproj.no_auto"
 
-run_codeql_database_create([], test_db="default-db", db=None, lang="csharp")
+run_codeql_database_create([], db=None, lang="csharp")
+check_diagnostics()