Add diagnostics rule for arbitrary build messages

Python: Add test of keyword argument with same name as positional-only parameter

.github/actions/cache-query-compilation/action.yml

@@ -9,7 +9,7 @@ inputs:
 outputs:
   cache-dir:
     description: "The directory where the cache was stored"
-    value: ${{ steps.fill-compilation-dir.outputs.compdir }}
+    value: ${{ steps.output-compilation-dir.outputs.compdir }}
 
 runs:
   using: composite
@@ -27,7 +27,9 @@ runs:
       if: ${{ github.event_name == 'pull_request' }}
       uses: actions/cache/restore@v3
       with:
-        path: '**/.cache'
+        path: |
+          **/.cache
+          ~/.codeql/compile-cache
         key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }}
         restore-keys: |
           codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
@@ -37,18 +39,111 @@ runs:
       if: ${{ github.event_name != 'pull_request' }}
       uses: actions/cache@v3
       with:
-        path: '**/.cache'
+        path: |
+          **/.cache
+          ~/.codeql/compile-cache
         key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
         restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
           codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
           codeql-compile-${{ inputs.key }}-main-
-    - name: Fill compilation cache directory
-      id: fill-compilation-dir
+    - name: Output-compilationdir
+      id: output-compilation-dir
       shell: bash
       run: |
-        # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-        node $GITHUB_WORKSPACE/.github/actions/cache-query-compilation/move-caches.js ${COMBINED_CACHE_DIR}
-
         echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
       env:
         COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
+    - name: Fill compilation cache directory
+      id: fill-compilation-dir
+      uses: actions/github-script@v6
+      env: 
+        COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
+      with:
+        script: |
+          // # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+          // mkdir -p ${COMBINED_CACHE_DIR}
+          // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+          // # copy the contents of the .cache folders into the combined cache folder.
+          // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+          // # clean up the .cache folders
+          // rm -rf **/.cache/*
+
+          const fs = require("fs");
+          const path = require("path");
+          const os = require("os");
+
+          // the first argv is the cache folder to create.
+          const COMBINED_CACHE_DIR = process.env.COMBINED_CACHE_DIR;
+
+          function* walkCaches(dir) {
+            const files = fs.readdirSync(dir, { withFileTypes: true });
+            for (const file of files) {
+              if (file.isDirectory()) {
+                const filePath = path.join(dir, file.name);
+                yield* walkCaches(filePath);
+                if (file.name === ".cache") {
+                  yield filePath;
+                }
+              }
+            }
+          }
+
+          async function copyDir(src, dest) {
+            for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
+              const srcPath = path.join(src, file.name);
+              const destPath = path.join(dest, file.name);
+              if (file.isDirectory()) {
+                if (!fs.existsSync(destPath)) {
+                  fs.mkdirSync(destPath);
+                }
+                await copyDir(srcPath, destPath);
+              } else {
+                await fs.promises.copyFile(srcPath, destPath);
+              }
+            }
+          }
+
+          async function main() {
+            const cacheDirs = [...walkCaches(".")];
+
+            for (const dir of cacheDirs) {
+              console.log(`Found .cache dir at ${dir}`);
+            }
+
+            const globalCacheDir = path.join(os.homedir(), ".codeql", "compile-cache");
+            if (fs.existsSync(globalCacheDir)) {
+              console.log("Found global home dir: " + globalCacheDir);
+              cacheDirs.push(globalCacheDir);
+            }
+
+            if (cacheDirs.length === 0) {
+              console.log("No cache dirs found");
+              return;
+            }
+
+            // mkdir -p ${COMBINED_CACHE_DIR}
+            fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
+
+            // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+            await Promise.all(
+              cacheDirs.map((cacheDir) =>
+                (async function () {
+                  await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
+                  await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
+                })()
+              )
+            );
+
+            // # copy the contents of the .cache folders into the combined cache folder.
+            // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+            await Promise.all(
+              cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
+            );
+
+            // # clean up the .cache folders
+            // rm -rf **/.cache/*
+            await Promise.all(
+              cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
+            );
+          }
+          main();

.github/actions/cache-query-compilation/move-caches.js

@@ -1,75 +0,0 @@
-// # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-// mkdir -p ${COMBINED_CACHE_DIR}
-// rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-// # copy the contents of the .cache folders into the combined cache folder.
-// cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-// # clean up the .cache folders
-// rm -rf **/.cache/*
-
-const fs = require("fs");
-const path = require("path");
-
-// the first argv is the cache folder to create.
-const COMBINED_CACHE_DIR = process.argv[2];
-
-function* walkCaches(dir) {
-  const files = fs.readdirSync(dir, { withFileTypes: true });
-  for (const file of files) {
-    if (file.isDirectory()) {
-      const filePath = path.join(dir, file.name);
-      yield* walkCaches(filePath);
-      if (file.name === ".cache") {
-        yield filePath;
-      }
-    }
-  }
-}
-
-async function copyDir(src, dest) {
-  for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
-    const srcPath = path.join(src, file.name);
-    const destPath = path.join(dest, file.name);
-    if (file.isDirectory()) {
-      if (!fs.existsSync(destPath)) {
-        fs.mkdirSync(destPath);
-      }
-      await copyDir(srcPath, destPath);
-    } else {
-      await fs.promises.copyFile(srcPath, destPath);
-    }
-  }
-}
-
-async function main() {
-  const cacheDirs = [...walkCaches(".")];
-
-  for (const dir of cacheDirs) {
-    console.log(`Found .cache dir at ${dir}`);
-  }
-
-  // mkdir -p ${COMBINED_CACHE_DIR}
-  fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
-
-  // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-  await Promise.all(
-    cacheDirs.map((cacheDir) =>
-      (async function () {
-        await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
-        await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
-      })()
-    )
-  );
-
-  // # copy the contents of the .cache folders into the combined cache folder.
-  // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-  await Promise.all(
-    cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
-  );
-
-  // # clean up the .cache folders
-  // rm -rf **/.cache/*
-  await Promise.all(
-    cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
-  );
-}
-main();

.github/workflows/check-qldoc.yml

@@ -26,9 +26,8 @@ jobs:
         shell: bash
         run: |
           EXIT_CODE=0
-          # TODO: remove the swift exception from the regex when we fix generated QLdoc
           # TODO: remove the shared exception from the regex when coverage of qlpacks without dbschemes is supported
-          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!(swift|shared))[a-z]*/ql/lib' || true; } | sort -u)"
+          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!(shared))[a-z]*/ql/lib' || true; } | sort -u)"
           for pack_dir in ${changed_lib_packs}; do
             lang="${pack_dir%/ql/lib}"
             codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"

.github/workflows/compile-queries.yml

@@ -24,14 +24,14 @@ jobs:
         with: 
           key: all-queries
       - name: check formatting
-        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
+        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 -n 3000 -P 10 codeql query format -q --check-only
       - name: compile queries - check-only
         # run with --check-only if running in a PR (github.sha != main)
         if : ${{ github.event_name == 'pull_request' }}
         shell: bash
-        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
       - name: compile queries - full
         # do full compile if running on main - this populates the cache
         if : ${{ github.event_name != 'pull_request' }}
         shell: bash
-        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/ql-for-ql-build.yml

@@ -5,13 +5,6 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
-    paths:
-      - "ql/**"
-      - "**.qll"
-      - "**.ql"
-      - "**.dbscheme"
-      - "**/qlpack.yml"
-      - ".github/workflows/ql-for-ql-build.yml"
 
 env:
   CARGO_TERM_COLOR: always
@@ -22,6 +15,8 @@ jobs:
     steps:
       ### Build the queries ###
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - name: Find codeql
         id: find-codeql
         uses: github/codeql-action/init@v2
@@ -34,7 +29,9 @@ jobs:
         id: cache-extractor
         uses: actions/cache@v3
         with:
-          path: ql/extractor-pack/
+          path: |
+            ql/extractor-pack/
+            ql/target/release/buramu
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
       - name: Cache cargo
         if: steps.cache-extractor.outputs.cache-hit != 'true'
@@ -57,6 +54,7 @@ jobs:
           key: run-ql-for-ql
       - name: Make database and analyze
         run: |
+          ./ql/target/release/buramu | tee deprecated.blame # Add a blame file for the extractor to parse.
           ${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB}
           ${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls  --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env: 
@@ -65,6 +63,7 @@ jobs:
           LGTM_INDEX_FILTERS: |
             exclude:ql/ql/test
             exclude:*/ql/lib/upgrades/
+            exclude:java/ql/integration-tests
       - name: Upload sarif to code-scanning
         uses: github/codeql-action/upload-sarif@v2
         with:

.github/workflows/swift.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "swift/**"
       - "misc/bazel/**"
+      - "misc/codegen/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**
@@ -19,6 +20,7 @@ on:
     paths:
       - "swift/**"
       - "misc/bazel/**"
+      - "misc/codegen/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**

.pre-commit-config.yaml

@@ -53,5 +53,5 @@ repos:
         name: Run Swift code generation unit tests
         files: ^swift/codegen/.*\.py$
         language: system
-        entry: bazel test //swift/codegen/test
+        entry: bazel test //misc/codegen/test
         pass_filenames: false

CODEOWNERS

@@ -2,10 +2,11 @@
 /csharp/ @github/codeql-csharp
 /go/ @github/codeql-go
 /java/ @github/codeql-java
-/javascript/ @github/codeql-javascript
-/python/ @github/codeql-python
-/ruby/ @github/codeql-ruby
+/javascript/ @github/codeql-dynamic
+/python/ @github/codeql-dynamic
+/ruby/ @github/codeql-dynamic
 /swift/ @github/codeql-swift
+/misc/codegen/ @github/codeql-swift
 /java/kotlin-extractor/ @github/codeql-kotlin
 /java/kotlin-explorer/ @github/codeql-kotlin
 

config/identical-files.json

@@ -1,6 +1,28 @@
 {
   "DataFlow Java/C++/C#/Go/Python/Ruby/Swift": [
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlow.qll",
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlow.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlow.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlow.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlow.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/DataFlow.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlow.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlow.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlow.qll"
+  ],
+  "DataFlowImpl Java/C++/C#/Go/Python/Ruby/Swift": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"
+  ],
+  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift Legacy Configuration": [
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl1.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl2.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl3.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl4.qll",
@@ -8,38 +30,38 @@
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl6.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForSerializability.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForOnActivityResult.qll",
-    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl1.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl4.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplLocal.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl1.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll",
-    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl1.qll",
     "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll",
     "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl1.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl2.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplForContentDataFlow.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl1.qll",
     "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl2.qll",
     "go/ql/lib/semmle/go/dataflow/internal/DataFlowImplForStringsNewReplacer.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl1.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplForRegExp.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl1.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForHttpClientLibraries.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForPathname.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl1.qll"
   ],
   "DataFlow Java/C++/C#/Go/Python/Ruby/Swift Common": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll",
@@ -52,7 +74,18 @@
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll"
   ],
-  "TaintTracking::Configuration Java/C++/C#/Go/Python/Ruby/Swift": [
+  "TaintTracking Java/C++/C#/Go/Python/Ruby/Swift": [
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTracking.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTracking.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTracking.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking1/TaintTracking.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/tainttracking1/TaintTracking.qll",
+    "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTracking.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking1/TaintTracking.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTracking.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/tainttracking1/TaintTracking.qll"
+  ],
+  "TaintTracking Legacy Configuration Java/C++/C#/Go/Python/Ruby/Swift": [
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -1,5 +1,6 @@
 using Xunit;
 using Semmle.Autobuild.Shared;
+using Semmle.Util;
 using System.Collections.Generic;
 using System;
 using System.Linq;
@@ -75,6 +76,15 @@ namespace Semmle.Autobuild.Cpp.Tests
             throw new ArgumentException("Missing RunProcess " + pattern);
         }
 
+        int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
+        {
+            var ret = (this as IBuildActions).RunProcess(cmd, args, workingDirectory, env, out var stdout);
+
+            stdout.ForEach(line => onOutput(line));
+
+            return ret;
+        }
+
         public IList<string> DirectoryDeleteIn = new List<string>();
 
         void IBuildActions.DirectoryDelete(string dir, bool recursive)
@@ -131,6 +141,14 @@ namespace Semmle.Autobuild.Cpp.Tests
 
         bool IBuildActions.IsWindows() => IsWindows;
 
+        public bool IsMacOs { get; set; }
+
+        bool IBuildActions.IsMacOs() => IsMacOs;
+
+        public bool IsArm { get; set; }
+
+        bool IBuildActions.IsArm() => IsArm;
+
         string IBuildActions.PathCombine(params string[] parts)
         {
             return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));
@@ -176,6 +194,15 @@ namespace Semmle.Autobuild.Cpp.Tests
             if (!DownloadFiles.Contains((address, fileName)))
                 throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");
         }
+
+        public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new TestDiagnosticWriter();
+    }
+
+    internal class TestDiagnosticWriter : IDiagnosticsWriter
+    {
+        public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
+
+        public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
     }
 
     /// <summary>
@@ -235,6 +262,7 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
+            Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
             Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";

cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs

@@ -1,4 +1,5 @@
 using Semmle.Autobuild.Shared;
+using Semmle.Util;
 
 namespace Semmle.Autobuild.Cpp
 {
@@ -21,7 +22,7 @@ namespace Semmle.Autobuild.Cpp
 
     public class CppAutobuilder : Autobuilder<CppAutobuildOptions>
     {
-        public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options) { }
+        public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options, new DiagnosticClassifier()) { }
 
         public override BuildScript GetBuildScript()
         {

cpp/ql/lib/change-notes/2023-03-02-dataflow-conf-module.md

@@ -0,0 +1,9 @@
+---
+category: majorAnalysis
+---
+* The main data flow and taint tracking APIs have been changed. The old APIs
+  remain in place for now and translate to the new through a
+  backwards-compatible wrapper. If multiple configurations are in scope
+  simultaneously, then this may affect results slightly. The new API is quite
+  similar to the old, but makes use of a configuration module instead of a
+  configuration class.

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/DataFlow.qll

@@ -22,5 +22,6 @@
 import cpp
 
 module DataFlow {
-  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlow
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl1
 }

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/TaintTracking.qll

@@ -19,5 +19,6 @@ import semmle.code.cpp.ir.dataflow.DataFlow
 import semmle.code.cpp.ir.dataflow.DataFlow2
 
 module TaintTracking {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.tainttracking1.TaintTracking
   import experimental.semmle.code.cpp.ir.dataflow.internal.tainttracking1.TaintTrackingImpl
 }

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlow.qll

@@ -0,0 +1,245 @@
+/**
+ * Provides an implementation of global (interprocedural) data flow. This file
+ * re-exports the local (intraprocedural) data flow analysis from
+ * `DataFlowImplSpecific::Public` and adds a global analysis, mainly exposed
+ * through the `Make` and `MakeWithState` modules.
+ */
+
+private import DataFlowImplCommon
+private import DataFlowImplSpecific::Private
+import DataFlowImplSpecific::Public
+import DataFlowImplCommonPublic
+private import DataFlowImpl
+
+/** An input configuration for data flow. */
+signature module ConfigSig {
+  /**
+   * Holds if `source` is a relevant data flow source.
+   */
+  predicate isSource(Node source);
+
+  /**
+   * Holds if `sink` is a relevant data flow sink.
+   */
+  predicate isSink(Node sink);
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  default predicate isBarrier(Node node) { none() }
+
+  /** Holds if data flow into `node` is prohibited. */
+  default predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  default predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  default predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  default predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  default int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  default FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  default predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  default predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (as it is in a `path-problem` query).
+   */
+  default predicate includeHiddenNodes() { none() }
+}
+
+/** An input configuration for data flow using flow state. */
+signature module StateConfigSig {
+  bindingset[this]
+  class FlowState;
+
+  /**
+   * Holds if `source` is a relevant data flow source with the given initial
+   * `state`.
+   */
+  predicate isSource(Node source, FlowState state);
+
+  /**
+   * Holds if `sink` is a relevant data flow sink accepting `state`.
+   */
+  predicate isSink(Node sink, FlowState state);
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  default predicate isBarrier(Node node) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited when the flow state is
+   * `state`.
+   */
+  predicate isBarrier(Node node, FlowState state);
+
+  /** Holds if data flow into `node` is prohibited. */
+  default predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  default predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  default predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
+   */
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2);
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  default predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  default int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  default FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  default predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  default predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (as it is in a `path-problem` query).
+   */
+  default predicate includeHiddenNodes() { none() }
+}
+
+/**
+ * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
+ * measured in approximate number of interprocedural steps.
+ */
+signature int explorationLimitSig();
+
+/**
+ * The output of a data flow computation.
+ */
+signature module DataFlowSig {
+  /**
+   * A `Node` augmented with a call context (except for sinks) and an access path.
+   * Only those `PathNode`s that are reachable from a source, and which can reach a sink, are generated.
+   */
+  class PathNode;
+
+  /**
+   * Holds if data can flow from `source` to `sink`.
+   *
+   * The corresponding paths are generated from the end-points and the graph
+   * included in the module `PathGraph`.
+   */
+  predicate hasFlowPath(PathNode source, PathNode sink);
+
+  /**
+   * Holds if data can flow from `source` to `sink`.
+   */
+  predicate hasFlow(Node source, Node sink);
+
+  /**
+   * Holds if data can flow from some source to `sink`.
+   */
+  predicate hasFlowTo(Node sink);
+
+  /**
+   * Holds if data can flow from some source to `sink`.
+   */
+  predicate hasFlowToExpr(DataFlowExpr sink);
+}
+
+/**
+ * Constructs a standard data flow computation.
+ */
+module Make<ConfigSig Config> implements DataFlowSig {
+  private module C implements FullStateConfigSig {
+    import DefaultState<Config>
+    import Config
+  }
+
+  import Impl<C>
+}
+
+/**
+ * Constructs a data flow computation using flow state.
+ */
+module MakeWithState<StateConfigSig Config> implements DataFlowSig {
+  private module C implements FullStateConfigSig {
+    import Config
+  }
+
+  import Impl<C>
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl1.qll

@@ -0,0 +1,396 @@
+/**
+ * DEPRECATED: Use `Make` and `MakeWithState` instead.
+ *
+ * Provides a `Configuration` class backwards-compatible interface to the data
+ * flow library.
+ */
+
+private import DataFlowImplCommon
+private import DataFlowImplSpecific::Private
+import DataFlowImplSpecific::Public
+private import DataFlowImpl
+import DataFlowImplCommonPublic
+import FlowStateString
+
+/**
+ * A configuration of interprocedural data flow analysis. This defines
+ * sources, sinks, and any other configurable aspect of the analysis. Each
+ * use of the global data flow library must define its own unique extension
+ * of this abstract class. To create a configuration, extend this class with
+ * a subclass whose characteristic predicate is a unique singleton string.
+ * For example, write
+ *
+ * ```ql
+ * class MyAnalysisConfiguration extends DataFlow::Configuration {
+ *   MyAnalysisConfiguration() { this = "MyAnalysisConfiguration" }
+ *   // Override `isSource` and `isSink`.
+ *   // Optionally override `isBarrier`.
+ *   // Optionally override `isAdditionalFlowStep`.
+ * }
+ * ```
+ * Conceptually, this defines a graph where the nodes are `DataFlow::Node`s and
+ * the edges are those data-flow steps that preserve the value of the node
+ * along with any additional edges defined by `isAdditionalFlowStep`.
+ * Specifying nodes in `isBarrier` will remove those nodes from the graph, and
+ * specifying nodes in `isBarrierIn` and/or `isBarrierOut` will remove in-going
+ * and/or out-going edges from those nodes, respectively.
+ *
+ * Then, to query whether there is flow between some `source` and `sink`,
+ * write
+ *
+ * ```ql
+ * exists(MyAnalysisConfiguration cfg | cfg.hasFlow(source, sink))
+ * ```
+ *
+ * Multiple configurations can coexist, but two classes extending
+ * `DataFlow::Configuration` should never depend on each other. One of them
+ * should instead depend on a `DataFlow2::Configuration`, a
+ * `DataFlow3::Configuration`, or a `DataFlow4::Configuration`.
+ */
+abstract class Configuration extends string {
+  bindingset[this]
+  Configuration() { any() }
+
+  /**
+   * Holds if `source` is a relevant data flow source.
+   */
+  predicate isSource(Node source) { none() }
+
+  /**
+   * Holds if `source` is a relevant data flow source with the given initial
+   * `state`.
+   */
+  predicate isSource(Node source, FlowState state) { none() }
+
+  /**
+   * Holds if `sink` is a relevant data flow sink.
+   */
+  predicate isSink(Node sink) { none() }
+
+  /**
+   * Holds if `sink` is a relevant data flow sink accepting `state`.
+   */
+  predicate isSink(Node sink, FlowState state) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  predicate isBarrier(Node node) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited when the flow state is
+   * `state`.
+   */
+  predicate isBarrier(Node node, FlowState state) { none() }
+
+  /** Holds if data flow into `node` is prohibited. */
+  predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * DEPRECATED: Use `isBarrier` and `BarrierGuard` module instead.
+   *
+   * Holds if data flow through nodes guarded by `guard` is prohibited.
+   */
+  deprecated predicate isBarrierGuard(BarrierGuard guard) { none() }
+
+  /**
+   * DEPRECATED: Use `isBarrier` and `BarrierGuard` module instead.
+   *
+   * Holds if data flow through nodes guarded by `guard` is prohibited when
+   * the flow state is `state`
+   */
+  deprecated predicate isBarrierGuard(BarrierGuard guard, FlowState state) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
+   */
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
+    none()
+  }
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if data may flow from `source` to `sink` for this configuration.
+   */
+  predicate hasFlow(Node source, Node sink) { hasFlow(source, sink, this) }
+
+  /**
+   * Holds if data may flow from `source` to `sink` for this configuration.
+   *
+   * The corresponding paths are generated from the end-points and the graph
+   * included in the module `PathGraph`.
+   */
+  predicate hasFlowPath(PathNode source, PathNode sink) { hasFlowPath(source, sink, this) }
+
+  /**
+   * Holds if data may flow from some source to `sink` for this configuration.
+   */
+  predicate hasFlowTo(Node sink) { hasFlowTo(sink, this) }
+
+  /**
+   * Holds if data may flow from some source to `sink` for this configuration.
+   */
+  predicate hasFlowToExpr(DataFlowExpr sink) { this.hasFlowTo(exprNode(sink)) }
+
+  /**
+   * DEPRECATED: Use `FlowExploration<explorationLimit>` instead.
+   *
+   * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
+   * measured in approximate number of interprocedural steps.
+   */
+  deprecated int explorationLimit() { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+}
+
+/**
+ * This class exists to prevent mutual recursion between the user-overridden
+ * member predicates of `Configuration` and the rest of the data-flow library.
+ * Good performance cannot be guaranteed in the presence of such recursion, so
+ * it should be replaced by using more than one copy of the data flow library.
+ */
+abstract private class ConfigurationRecursionPrevention extends Configuration {
+  bindingset[this]
+  ConfigurationRecursionPrevention() { any() }
+
+  override predicate hasFlow(Node source, Node sink) {
+    strictcount(Node n | this.isSource(n)) < 0
+    or
+    strictcount(Node n | this.isSource(n, _)) < 0
+    or
+    strictcount(Node n | this.isSink(n)) < 0
+    or
+    strictcount(Node n | this.isSink(n, _)) < 0
+    or
+    strictcount(Node n1, Node n2 | this.isAdditionalFlowStep(n1, n2)) < 0
+    or
+    strictcount(Node n1, Node n2 | this.isAdditionalFlowStep(n1, _, n2, _)) < 0
+    or
+    super.hasFlow(source, sink)
+  }
+}
+
+/** A bridge class to access the deprecated `isBarrierGuard`. */
+private class BarrierGuardGuardedNodeBridge extends Unit {
+  abstract predicate guardedNode(Node n, Configuration config);
+
+  abstract predicate guardedNode(Node n, FlowState state, Configuration config);
+}
+
+private class BarrierGuardGuardedNode extends BarrierGuardGuardedNodeBridge {
+  deprecated override predicate guardedNode(Node n, Configuration config) {
+    exists(BarrierGuard g |
+      config.isBarrierGuard(g) and
+      n = g.getAGuardedNode()
+    )
+  }
+
+  deprecated override predicate guardedNode(Node n, FlowState state, Configuration config) {
+    exists(BarrierGuard g |
+      config.isBarrierGuard(g, state) and
+      n = g.getAGuardedNode()
+    )
+  }
+}
+
+private FlowState relevantState(Configuration config) {
+  config.isSource(_, result) or
+  config.isSink(_, result) or
+  config.isBarrier(_, result) or
+  config.isAdditionalFlowStep(_, result, _, _) or
+  config.isAdditionalFlowStep(_, _, _, result)
+}
+
+private newtype TConfigState =
+  TMkConfigState(Configuration config, FlowState state) {
+    state = relevantState(config) or state instanceof FlowStateEmpty
+  }
+
+private Configuration getConfig(TConfigState state) { state = TMkConfigState(result, _) }
+
+private FlowState getState(TConfigState state) { state = TMkConfigState(_, result) }
+
+private predicate singleConfiguration() { 1 = strictcount(Configuration c) }
+
+private module Config implements FullStateConfigSig {
+  class FlowState = TConfigState;
+
+  predicate isSource(Node source, FlowState state) {
+    getConfig(state).isSource(source, getState(state))
+    or
+    getConfig(state).isSource(source) and getState(state) instanceof FlowStateEmpty
+  }
+
+  predicate isSink(Node sink, FlowState state) {
+    getConfig(state).isSink(sink, getState(state))
+    or
+    getConfig(state).isSink(sink) and getState(state) instanceof FlowStateEmpty
+  }
+
+  predicate isBarrier(Node node) { none() }
+
+  predicate isBarrier(Node node, FlowState state) {
+    getConfig(state).isBarrier(node, getState(state)) or
+    getConfig(state).isBarrier(node) or
+    any(BarrierGuardGuardedNodeBridge b).guardedNode(node, getState(state), getConfig(state)) or
+    any(BarrierGuardGuardedNodeBridge b).guardedNode(node, getConfig(state))
+  }
+
+  predicate isBarrierIn(Node node) { any(Configuration config).isBarrierIn(node) }
+
+  predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
+
+  predicate isAdditionalFlowStep(Node node1, Node node2) {
+    singleConfiguration() and
+    any(Configuration config).isAdditionalFlowStep(node1, node2)
+  }
+
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
+    getConfig(state1).isAdditionalFlowStep(node1, getState(state1), node2, getState(state2)) and
+    getConfig(state2) = getConfig(state1)
+    or
+    not singleConfiguration() and
+    getConfig(state1).isAdditionalFlowStep(node1, node2) and
+    state2 = state1
+  }
+
+  predicate allowImplicitRead(Node node, ContentSet c) {
+    any(Configuration config).allowImplicitRead(node, c)
+  }
+
+  int fieldFlowBranchLimit() { result = min(any(Configuration config).fieldFlowBranchLimit()) }
+
+  FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
+
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
+  predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
+}
+
+private import Impl<Config> as I
+import I
+
+/**
+ * A `Node` augmented with a call context (except for sinks), an access path, and a configuration.
+ * Only those `PathNode`s that are reachable from a source, and which can reach a sink, are generated.
+ */
+class PathNode instanceof I::PathNode {
+  /** Gets a textual representation of this element. */
+  final string toString() { result = super.toString() }
+
+  /**
+   * Gets a textual representation of this element, including a textual
+   * representation of the call context.
+   */
+  final string toStringWithContext() { result = super.toStringWithContext() }
+
+  /**
+   * Holds if this element is at the specified location.
+   * The location spans column `startcolumn` of line `startline` to
+   * column `endcolumn` of line `endline` in file `filepath`.
+   * For more information, see
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
+   */
+  final predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    super.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+  }
+
+  /** Gets the underlying `Node`. */
+  final Node getNode() { result = super.getNode() }
+
+  /** Gets the `FlowState` of this node. */
+  final FlowState getState() { result = getState(super.getState()) }
+
+  /** Gets the associated configuration. */
+  final Configuration getConfiguration() { result = getConfig(super.getState()) }
+
+  /** Gets a successor of this node, if any. */
+  final PathNode getASuccessor() { result = super.getASuccessor() }
+
+  /** Holds if this node is a source. */
+  final predicate isSource() { super.isSource() }
+
+  /** Holds if this node is a grouping of source nodes. */
+  final predicate isSourceGroup(string group) { super.isSourceGroup(group) }
+
+  /** Holds if this node is a grouping of sink nodes. */
+  final predicate isSinkGroup(string group) { super.isSinkGroup(group) }
+}
+
+private predicate hasFlow(Node source, Node sink, Configuration config) {
+  exists(PathNode source0, PathNode sink0 |
+    hasFlowPath(source0, sink0, config) and
+    source0.getNode() = source and
+    sink0.getNode() = sink
+  )
+}
+
+private predicate hasFlowPath(PathNode source, PathNode sink, Configuration config) {
+  hasFlowPath(source, sink) and source.getConfiguration() = config
+}
+
+private predicate hasFlowTo(Node sink, Configuration config) { hasFlow(_, sink, config) }
+
+predicate flowsTo = hasFlow/3;

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll

@@ -3,15 +3,18 @@ private import DataFlowImplSpecific::Public
 import Cached
 
 module DataFlowImplCommonPublic {
-  /** A state value to track during data flow. */
-  class FlowState = string;
+  /** Provides `FlowState = string`. */
+  module FlowStateString {
+    /** A state value to track during data flow. */
+    class FlowState = string;
 
-  /**
-   * The default state, which is used when the state is unspecified for a source
-   * or a sink.
-   */
-  class FlowStateEmpty extends FlowState {
-    FlowStateEmpty() { this = "" }
+    /**
+     * The default state, which is used when the state is unspecified for a source
+     * or a sink.
+     */
+    class FlowStateEmpty extends FlowState {
+      FlowStateEmpty() { this = "" }
+    }
   }
 
   private newtype TFlowFeature =

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll

@@ -565,3 +565,12 @@ private class MyConsistencyConfiguration extends Consistency::ConsistencyConfigu
     any()
   }
 }
+
+/**
+ * Gets an additional term that is added to the `join` and `branch` computations to reflect
+ * an additional forward or backwards branching factor that is not taken into account
+ * when calculating the (virtual) dispatch cost.
+ *
+ * Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter.
+ */
+int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p) { none() }

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/PrintIRLocalFlow.qll

@@ -97,23 +97,23 @@ private string getNodeProperty(DataFlow::Node node, string key) {
     |
       kind, ", "
     )
-  or
-  // Is there partial flow from a source to this node?
-  // This property will only be emitted if partial flow is enabled by overriding
-  // `DataFlow::Configuration::explorationLimit()`.
-  key = "pflow" and
-  result =
-    strictconcat(DataFlow::PartialPathNode sourceNode, DataFlow::PartialPathNode destNode, int dist,
-      int order1, int order2 |
-      any(DataFlow::Configuration cfg).hasPartialFlow(sourceNode, destNode, dist) and
-      destNode.getNode() = node and
-      // Only print flow from a source in the same function.
-      sourceNode.getNode().getEnclosingCallable() = node.getEnclosingCallable()
-    |
-      nodeId(sourceNode.getNode(), order1, order2) + "+" + dist.toString(), ", "
-      order by
-        order1, order2, dist desc
-    )
+  // or
+  // // Is there partial flow from a source to this node?
+  // // This property will only be emitted if partial flow is enabled by overriding
+  // // `DataFlow::Configuration::explorationLimit()`.
+  // key = "pflow" and
+  // result =
+  //   strictconcat(DataFlow::PartialPathNode sourceNode, DataFlow::PartialPathNode destNode, int dist,
+  //     int order1, int order2 |
+  //     any(DataFlow::Configuration cfg).hasPartialFlow(sourceNode, destNode, dist) and
+  //     destNode.getNode() = node and
+  //     // Only print flow from a source in the same function.
+  //     sourceNode.getNode().getEnclosingCallable() = node.getEnclosingCallable()
+  //   |
+  //     nodeId(sourceNode.getNode(), order1, order2) + "+" + dist.toString(), ", "
+  //     order by
+  //       order1, order2, dist desc
+  //   )
 }
 
 /**

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTracking.qll

@@ -0,0 +1,63 @@
+/**
+ * Provides classes for performing local (intra-procedural) and
+ * global (inter-procedural) taint-tracking analyses.
+ */
+
+import TaintTrackingParameter::Public
+private import TaintTrackingParameter::Private
+
+private module AddTaintDefaults<DataFlowInternal::FullStateConfigSig Config> implements
+DataFlowInternal::FullStateConfigSig {
+  import Config
+
+  predicate isBarrier(DataFlow::Node node) {
+    Config::isBarrier(node) or defaultTaintSanitizer(node)
+  }
+
+  predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
+    Config::isAdditionalFlowStep(node1, node2) or
+    defaultAdditionalTaintStep(node1, node2)
+  }
+
+  predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c) {
+    Config::allowImplicitRead(node, c)
+    or
+    (
+      Config::isSink(node, _) or
+      Config::isAdditionalFlowStep(node, _) or
+      Config::isAdditionalFlowStep(node, _, _, _)
+    ) and
+    defaultImplicitTaintRead(node, c)
+  }
+}
+
+/**
+ * Constructs a standard taint tracking computation.
+ */
+module Make<DataFlow::ConfigSig Config> implements DataFlow::DataFlowSig {
+  private module Config0 implements DataFlowInternal::FullStateConfigSig {
+    import DataFlowInternal::DefaultState<Config>
+    import Config
+  }
+
+  private module C implements DataFlowInternal::FullStateConfigSig {
+    import AddTaintDefaults<Config0>
+  }
+
+  import DataFlowInternal::Impl<C>
+}
+
+/**
+ * Constructs a taint tracking computation using flow state.
+ */
+module MakeWithState<DataFlow::StateConfigSig Config> implements DataFlow::DataFlowSig {
+  private module Config0 implements DataFlowInternal::FullStateConfigSig {
+    import Config
+  }
+
+  private module C implements DataFlowInternal::FullStateConfigSig {
+    import AddTaintDefaults<Config0>
+  }
+
+  import DataFlowInternal::Impl<C>
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingParameter.qll

@@ -2,4 +2,5 @@ import experimental.semmle.code.cpp.ir.dataflow.internal.TaintTrackingUtil as Pu
 
 module Private {
   import experimental.semmle.code.cpp.ir.dataflow.DataFlow::DataFlow as DataFlow
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl as DataFlowInternal
 }

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.5.3
+version: 0.5.4-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/Declaration.qll

@@ -68,7 +68,9 @@ class Declaration extends Locatable, @declaration {
    * Holds if this declaration has the fully-qualified name `qualifiedName`.
    * See `getQualifiedName`.
    */
-  predicate hasQualifiedName(string qualifiedName) { this.getQualifiedName() = qualifiedName }
+  deprecated predicate hasQualifiedName(string qualifiedName) {
+    this.getQualifiedName() = qualifiedName
+  }
 
   /**
    * Holds if this declaration has a fully-qualified name with a name-space
@@ -184,7 +186,7 @@ class Declaration extends Locatable, @declaration {
   predicate hasDefinition() { exists(this.getDefinition()) }
 
   /** DEPRECATED: Use `hasDefinition` instead. */
-  predicate isDefined() { this.hasDefinition() }
+  deprecated predicate isDefined() { this.hasDefinition() }
 
   /** Gets the preferred location of this declaration, if any. */
   override Location getLocation() { none() }

cpp/ql/lib/semmle/code/cpp/Function.qll

@@ -41,7 +41,7 @@ class Function extends Declaration, ControlFlowNode, AccessHolder, @function {
    * `min<int>(int, int) -> int`, and the full signature of the uninstantiated
    * template on the first line would be `min<T>(T, T) -> T`.
    */
-  string getFullSignature() {
+  deprecated string getFullSignature() {
     exists(string name, string templateArgs, string args |
       result = name + templateArgs + args + " -> " + this.getType().toString() and
       name = this.getQualifiedName() and

cpp/ql/lib/semmle/code/cpp/commons/Alloc.qll

@@ -12,7 +12,9 @@ predicate freeFunction(Function f, int argNum) { argNum = f.(DeallocationFunctio
  *
  * DEPRECATED: Use `DeallocationExpr` instead (this also includes `delete` expressions).
  */
-predicate freeCall(FunctionCall fc, Expr arg) { arg = fc.(DeallocationExpr).getFreedExpr() }
+deprecated predicate freeCall(FunctionCall fc, Expr arg) {
+  arg = fc.(DeallocationExpr).getFreedExpr()
+}
 
 /**
  * Is e some kind of allocation or deallocation (`new`, `alloc`, `realloc`, `delete`, `free` etc)?

cpp/ql/lib/semmle/code/cpp/dataflow/DataFlow.qll

@@ -20,5 +20,6 @@
 import cpp
 
 module DataFlow {
-  import semmle.code.cpp.dataflow.internal.DataFlowImpl
+  import semmle.code.cpp.dataflow.internal.DataFlow
+  import semmle.code.cpp.dataflow.internal.DataFlowImpl1
 }

cpp/ql/lib/semmle/code/cpp/dataflow/TaintTracking.qll

@@ -19,5 +19,6 @@ import semmle.code.cpp.dataflow.DataFlow
 import semmle.code.cpp.dataflow.DataFlow2
 
 module TaintTracking {
+  import semmle.code.cpp.dataflow.internal.tainttracking1.TaintTracking
   import semmle.code.cpp.dataflow.internal.tainttracking1.TaintTrackingImpl
 }

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlow.qll

@@ -0,0 +1,245 @@
+/**
+ * Provides an implementation of global (interprocedural) data flow. This file
+ * re-exports the local (intraprocedural) data flow analysis from
+ * `DataFlowImplSpecific::Public` and adds a global analysis, mainly exposed
+ * through the `Make` and `MakeWithState` modules.
+ */
+
+private import DataFlowImplCommon
+private import DataFlowImplSpecific::Private
+import DataFlowImplSpecific::Public
+import DataFlowImplCommonPublic
+private import DataFlowImpl
+
+/** An input configuration for data flow. */
+signature module ConfigSig {
+  /**
+   * Holds if `source` is a relevant data flow source.
+   */
+  predicate isSource(Node source);
+
+  /**
+   * Holds if `sink` is a relevant data flow sink.
+   */
+  predicate isSink(Node sink);
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  default predicate isBarrier(Node node) { none() }
+
+  /** Holds if data flow into `node` is prohibited. */
+  default predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  default predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  default predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  default predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  default int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  default FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  default predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  default predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (as it is in a `path-problem` query).
+   */
+  default predicate includeHiddenNodes() { none() }
+}
+
+/** An input configuration for data flow using flow state. */
+signature module StateConfigSig {
+  bindingset[this]
+  class FlowState;
+
+  /**
+   * Holds if `source` is a relevant data flow source with the given initial
+   * `state`.
+   */
+  predicate isSource(Node source, FlowState state);
+
+  /**
+   * Holds if `sink` is a relevant data flow sink accepting `state`.
+   */
+  predicate isSink(Node sink, FlowState state);
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  default predicate isBarrier(Node node) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited when the flow state is
+   * `state`.
+   */
+  predicate isBarrier(Node node, FlowState state);
+
+  /** Holds if data flow into `node` is prohibited. */
+  default predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  default predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  default predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
+   */
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2);
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  default predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  default int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  default FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  default predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  default predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (as it is in a `path-problem` query).
+   */
+  default predicate includeHiddenNodes() { none() }
+}
+
+/**
+ * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
+ * measured in approximate number of interprocedural steps.
+ */
+signature int explorationLimitSig();
+
+/**
+ * The output of a data flow computation.
+ */
+signature module DataFlowSig {
+  /**
+   * A `Node` augmented with a call context (except for sinks) and an access path.
+   * Only those `PathNode`s that are reachable from a source, and which can reach a sink, are generated.
+   */
+  class PathNode;
+
+  /**
+   * Holds if data can flow from `source` to `sink`.
+   *
+   * The corresponding paths are generated from the end-points and the graph
+   * included in the module `PathGraph`.
+   */
+  predicate hasFlowPath(PathNode source, PathNode sink);
+
+  /**
+   * Holds if data can flow from `source` to `sink`.
+   */
+  predicate hasFlow(Node source, Node sink);
+
+  /**
+   * Holds if data can flow from some source to `sink`.
+   */
+  predicate hasFlowTo(Node sink);
+
+  /**
+   * Holds if data can flow from some source to `sink`.
+   */
+  predicate hasFlowToExpr(DataFlowExpr sink);
+}
+
+/**
+ * Constructs a standard data flow computation.
+ */
+module Make<ConfigSig Config> implements DataFlowSig {
+  private module C implements FullStateConfigSig {
+    import DefaultState<Config>
+    import Config
+  }
+
+  import Impl<C>
+}
+
+/**
+ * Constructs a data flow computation using flow state.
+ */
+module MakeWithState<StateConfigSig Config> implements DataFlowSig {
+  private module C implements FullStateConfigSig {
+    import Config
+  }
+
+  import Impl<C>
+}

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl1.qll

@@ -0,0 +1,396 @@
+/**
+ * DEPRECATED: Use `Make` and `MakeWithState` instead.
+ *
+ * Provides a `Configuration` class backwards-compatible interface to the data
+ * flow library.
+ */
+
+private import DataFlowImplCommon
+private import DataFlowImplSpecific::Private
+import DataFlowImplSpecific::Public
+private import DataFlowImpl
+import DataFlowImplCommonPublic
+import FlowStateString
+
+/**
+ * A configuration of interprocedural data flow analysis. This defines
+ * sources, sinks, and any other configurable aspect of the analysis. Each
+ * use of the global data flow library must define its own unique extension
+ * of this abstract class. To create a configuration, extend this class with
+ * a subclass whose characteristic predicate is a unique singleton string.
+ * For example, write
+ *
+ * ```ql
+ * class MyAnalysisConfiguration extends DataFlow::Configuration {
+ *   MyAnalysisConfiguration() { this = "MyAnalysisConfiguration" }
+ *   // Override `isSource` and `isSink`.
+ *   // Optionally override `isBarrier`.
+ *   // Optionally override `isAdditionalFlowStep`.
+ * }
+ * ```
+ * Conceptually, this defines a graph where the nodes are `DataFlow::Node`s and
+ * the edges are those data-flow steps that preserve the value of the node
+ * along with any additional edges defined by `isAdditionalFlowStep`.
+ * Specifying nodes in `isBarrier` will remove those nodes from the graph, and
+ * specifying nodes in `isBarrierIn` and/or `isBarrierOut` will remove in-going
+ * and/or out-going edges from those nodes, respectively.
+ *
+ * Then, to query whether there is flow between some `source` and `sink`,
+ * write
+ *
+ * ```ql
+ * exists(MyAnalysisConfiguration cfg | cfg.hasFlow(source, sink))
+ * ```
+ *
+ * Multiple configurations can coexist, but two classes extending
+ * `DataFlow::Configuration` should never depend on each other. One of them
+ * should instead depend on a `DataFlow2::Configuration`, a
+ * `DataFlow3::Configuration`, or a `DataFlow4::Configuration`.
+ */
+abstract class Configuration extends string {
+  bindingset[this]
+  Configuration() { any() }
+
+  /**
+   * Holds if `source` is a relevant data flow source.
+   */
+  predicate isSource(Node source) { none() }
+
+  /**
+   * Holds if `source` is a relevant data flow source with the given initial
+   * `state`.
+   */
+  predicate isSource(Node source, FlowState state) { none() }
+
+  /**
+   * Holds if `sink` is a relevant data flow sink.
+   */
+  predicate isSink(Node sink) { none() }
+
+  /**
+   * Holds if `sink` is a relevant data flow sink accepting `state`.
+   */
+  predicate isSink(Node sink, FlowState state) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  predicate isBarrier(Node node) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited when the flow state is
+   * `state`.
+   */
+  predicate isBarrier(Node node, FlowState state) { none() }
+
+  /** Holds if data flow into `node` is prohibited. */
+  predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * DEPRECATED: Use `isBarrier` and `BarrierGuard` module instead.
+   *
+   * Holds if data flow through nodes guarded by `guard` is prohibited.
+   */
+  deprecated predicate isBarrierGuard(BarrierGuard guard) { none() }
+
+  /**
+   * DEPRECATED: Use `isBarrier` and `BarrierGuard` module instead.
+   *
+   * Holds if data flow through nodes guarded by `guard` is prohibited when
+   * the flow state is `state`
+   */
+  deprecated predicate isBarrierGuard(BarrierGuard guard, FlowState state) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
+   */
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
+    none()
+  }
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if data may flow from `source` to `sink` for this configuration.
+   */
+  predicate hasFlow(Node source, Node sink) { hasFlow(source, sink, this) }
+
+  /**
+   * Holds if data may flow from `source` to `sink` for this configuration.
+   *
+   * The corresponding paths are generated from the end-points and the graph
+   * included in the module `PathGraph`.
+   */
+  predicate hasFlowPath(PathNode source, PathNode sink) { hasFlowPath(source, sink, this) }
+
+  /**
+   * Holds if data may flow from some source to `sink` for this configuration.
+   */
+  predicate hasFlowTo(Node sink) { hasFlowTo(sink, this) }
+
+  /**
+   * Holds if data may flow from some source to `sink` for this configuration.
+   */
+  predicate hasFlowToExpr(DataFlowExpr sink) { this.hasFlowTo(exprNode(sink)) }
+
+  /**
+   * DEPRECATED: Use `FlowExploration<explorationLimit>` instead.
+   *
+   * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
+   * measured in approximate number of interprocedural steps.
+   */
+  deprecated int explorationLimit() { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+}
+
+/**
+ * This class exists to prevent mutual recursion between the user-overridden
+ * member predicates of `Configuration` and the rest of the data-flow library.
+ * Good performance cannot be guaranteed in the presence of such recursion, so
+ * it should be replaced by using more than one copy of the data flow library.
+ */
+abstract private class ConfigurationRecursionPrevention extends Configuration {
+  bindingset[this]
+  ConfigurationRecursionPrevention() { any() }
+
+  override predicate hasFlow(Node source, Node sink) {
+    strictcount(Node n | this.isSource(n)) < 0
+    or
+    strictcount(Node n | this.isSource(n, _)) < 0
+    or
+    strictcount(Node n | this.isSink(n)) < 0
+    or
+    strictcount(Node n | this.isSink(n, _)) < 0
+    or
+    strictcount(Node n1, Node n2 | this.isAdditionalFlowStep(n1, n2)) < 0
+    or
+    strictcount(Node n1, Node n2 | this.isAdditionalFlowStep(n1, _, n2, _)) < 0
+    or
+    super.hasFlow(source, sink)
+  }
+}
+
+/** A bridge class to access the deprecated `isBarrierGuard`. */
+private class BarrierGuardGuardedNodeBridge extends Unit {
+  abstract predicate guardedNode(Node n, Configuration config);
+
+  abstract predicate guardedNode(Node n, FlowState state, Configuration config);
+}
+
+private class BarrierGuardGuardedNode extends BarrierGuardGuardedNodeBridge {
+  deprecated override predicate guardedNode(Node n, Configuration config) {
+    exists(BarrierGuard g |
+      config.isBarrierGuard(g) and
+      n = g.getAGuardedNode()
+    )
+  }
+
+  deprecated override predicate guardedNode(Node n, FlowState state, Configuration config) {
+    exists(BarrierGuard g |
+      config.isBarrierGuard(g, state) and
+      n = g.getAGuardedNode()
+    )
+  }
+}
+
+private FlowState relevantState(Configuration config) {
+  config.isSource(_, result) or
+  config.isSink(_, result) or
+  config.isBarrier(_, result) or
+  config.isAdditionalFlowStep(_, result, _, _) or
+  config.isAdditionalFlowStep(_, _, _, result)
+}
+
+private newtype TConfigState =
+  TMkConfigState(Configuration config, FlowState state) {
+    state = relevantState(config) or state instanceof FlowStateEmpty
+  }
+
+private Configuration getConfig(TConfigState state) { state = TMkConfigState(result, _) }
+
+private FlowState getState(TConfigState state) { state = TMkConfigState(_, result) }
+
+private predicate singleConfiguration() { 1 = strictcount(Configuration c) }
+
+private module Config implements FullStateConfigSig {
+  class FlowState = TConfigState;
+
+  predicate isSource(Node source, FlowState state) {
+    getConfig(state).isSource(source, getState(state))
+    or
+    getConfig(state).isSource(source) and getState(state) instanceof FlowStateEmpty
+  }
+
+  predicate isSink(Node sink, FlowState state) {
+    getConfig(state).isSink(sink, getState(state))
+    or
+    getConfig(state).isSink(sink) and getState(state) instanceof FlowStateEmpty
+  }
+
+  predicate isBarrier(Node node) { none() }
+
+  predicate isBarrier(Node node, FlowState state) {
+    getConfig(state).isBarrier(node, getState(state)) or
+    getConfig(state).isBarrier(node) or
+    any(BarrierGuardGuardedNodeBridge b).guardedNode(node, getState(state), getConfig(state)) or
+    any(BarrierGuardGuardedNodeBridge b).guardedNode(node, getConfig(state))
+  }
+
+  predicate isBarrierIn(Node node) { any(Configuration config).isBarrierIn(node) }
+
+  predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
+
+  predicate isAdditionalFlowStep(Node node1, Node node2) {
+    singleConfiguration() and
+    any(Configuration config).isAdditionalFlowStep(node1, node2)
+  }
+
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
+    getConfig(state1).isAdditionalFlowStep(node1, getState(state1), node2, getState(state2)) and
+    getConfig(state2) = getConfig(state1)
+    or
+    not singleConfiguration() and
+    getConfig(state1).isAdditionalFlowStep(node1, node2) and
+    state2 = state1
+  }
+
+  predicate allowImplicitRead(Node node, ContentSet c) {
+    any(Configuration config).allowImplicitRead(node, c)
+  }
+
+  int fieldFlowBranchLimit() { result = min(any(Configuration config).fieldFlowBranchLimit()) }
+
+  FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
+
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
+  predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
+}
+
+private import Impl<Config> as I
+import I
+
+/**
+ * A `Node` augmented with a call context (except for sinks), an access path, and a configuration.
+ * Only those `PathNode`s that are reachable from a source, and which can reach a sink, are generated.
+ */
+class PathNode instanceof I::PathNode {
+  /** Gets a textual representation of this element. */
+  final string toString() { result = super.toString() }
+
+  /**
+   * Gets a textual representation of this element, including a textual
+   * representation of the call context.
+   */
+  final string toStringWithContext() { result = super.toStringWithContext() }
+
+  /**
+   * Holds if this element is at the specified location.
+   * The location spans column `startcolumn` of line `startline` to
+   * column `endcolumn` of line `endline` in file `filepath`.
+   * For more information, see
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
+   */
+  final predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    super.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+  }
+
+  /** Gets the underlying `Node`. */
+  final Node getNode() { result = super.getNode() }
+
+  /** Gets the `FlowState` of this node. */
+  final FlowState getState() { result = getState(super.getState()) }
+
+  /** Gets the associated configuration. */
+  final Configuration getConfiguration() { result = getConfig(super.getState()) }
+
+  /** Gets a successor of this node, if any. */
+  final PathNode getASuccessor() { result = super.getASuccessor() }
+
+  /** Holds if this node is a source. */
+  final predicate isSource() { super.isSource() }
+
+  /** Holds if this node is a grouping of source nodes. */
+  final predicate isSourceGroup(string group) { super.isSourceGroup(group) }
+
+  /** Holds if this node is a grouping of sink nodes. */
+  final predicate isSinkGroup(string group) { super.isSinkGroup(group) }
+}
+
+private predicate hasFlow(Node source, Node sink, Configuration config) {
+  exists(PathNode source0, PathNode sink0 |
+    hasFlowPath(source0, sink0, config) and
+    source0.getNode() = source and
+    sink0.getNode() = sink
+  )
+}
+
+private predicate hasFlowPath(PathNode source, PathNode sink, Configuration config) {
+  hasFlowPath(source, sink) and source.getConfiguration() = config
+}
+
+private predicate hasFlowTo(Node sink, Configuration config) { hasFlow(_, sink, config) }
+
+predicate flowsTo = hasFlow/3;

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll

@@ -3,15 +3,18 @@ private import DataFlowImplSpecific::Public
 import Cached
 
 module DataFlowImplCommonPublic {
-  /** A state value to track during data flow. */
-  class FlowState = string;
+  /** Provides `FlowState = string`. */
+  module FlowStateString {
+    /** A state value to track during data flow. */
+    class FlowState = string;
 
-  /**
-   * The default state, which is used when the state is unspecified for a source
-   * or a sink.
-   */
-  class FlowStateEmpty extends FlowState {
-    FlowStateEmpty() { this = "" }
+    /**
+     * The default state, which is used when the state is unspecified for a source
+     * or a sink.
+     */
+    class FlowStateEmpty extends FlowState {
+      FlowStateEmpty() { this = "" }
+    }
   }
 
   private newtype TFlowFeature =

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowPrivate.qll

@@ -318,3 +318,12 @@ private class MyConsistencyConfiguration extends Consistency::ConsistencyConfigu
     // consistency alerts enough that most of them are interesting.
   }
 }
+
+/**
+ * Gets an additional term that is added to the `join` and `branch` computations to reflect
+ * an additional forward or backwards branching factor that is not taken into account
+ * when calculating the (virtual) dispatch cost.
+ *
+ * Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter.
+ */
+int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p) { none() }

cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTracking.qll

@@ -0,0 +1,63 @@
+/**
+ * Provides classes for performing local (intra-procedural) and
+ * global (inter-procedural) taint-tracking analyses.
+ */
+
+import TaintTrackingParameter::Public
+private import TaintTrackingParameter::Private
+
+private module AddTaintDefaults<DataFlowInternal::FullStateConfigSig Config> implements
+DataFlowInternal::FullStateConfigSig {
+  import Config
+
+  predicate isBarrier(DataFlow::Node node) {
+    Config::isBarrier(node) or defaultTaintSanitizer(node)
+  }
+
+  predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
+    Config::isAdditionalFlowStep(node1, node2) or
+    defaultAdditionalTaintStep(node1, node2)
+  }
+
+  predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c) {
+    Config::allowImplicitRead(node, c)
+    or
+    (
+      Config::isSink(node, _) or
+      Config::isAdditionalFlowStep(node, _) or
+      Config::isAdditionalFlowStep(node, _, _, _)
+    ) and
+    defaultImplicitTaintRead(node, c)
+  }
+}
+
+/**
+ * Constructs a standard taint tracking computation.
+ */
+module Make<DataFlow::ConfigSig Config> implements DataFlow::DataFlowSig {
+  private module Config0 implements DataFlowInternal::FullStateConfigSig {
+    import DataFlowInternal::DefaultState<Config>
+    import Config
+  }
+
+  private module C implements DataFlowInternal::FullStateConfigSig {
+    import AddTaintDefaults<Config0>
+  }
+
+  import DataFlowInternal::Impl<C>
+}
+
+/**
+ * Constructs a taint tracking computation using flow state.
+ */
+module MakeWithState<DataFlow::StateConfigSig Config> implements DataFlow::DataFlowSig {
+  private module Config0 implements DataFlowInternal::FullStateConfigSig {
+    import Config
+  }
+
+  private module C implements DataFlowInternal::FullStateConfigSig {
+    import AddTaintDefaults<Config0>
+  }
+
+  import DataFlowInternal::Impl<C>
+}

cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingParameter.qll

@@ -2,4 +2,5 @@ import semmle.code.cpp.dataflow.internal.TaintTrackingUtil as Public
 
 module Private {
   import semmle.code.cpp.dataflow.DataFlow::DataFlow as DataFlow
+  import semmle.code.cpp.dataflow.internal.DataFlowImpl as DataFlowInternal
 }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/DataFlow.qll

@@ -22,5 +22,6 @@
 import cpp
 
 module DataFlow {
-  import semmle.code.cpp.ir.dataflow.internal.DataFlowImpl
+  import semmle.code.cpp.ir.dataflow.internal.DataFlow
+  import semmle.code.cpp.ir.dataflow.internal.DataFlowImpl1
 }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/TaintTracking.qll

@@ -19,5 +19,6 @@ import semmle.code.cpp.ir.dataflow.DataFlow
 import semmle.code.cpp.ir.dataflow.DataFlow2
 
 module TaintTracking {
+  import semmle.code.cpp.ir.dataflow.internal.tainttracking1.TaintTracking
   import semmle.code.cpp.ir.dataflow.internal.tainttracking1.TaintTrackingImpl
 }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlow.qll

@@ -0,0 +1,245 @@
+/**
+ * Provides an implementation of global (interprocedural) data flow. This file
+ * re-exports the local (intraprocedural) data flow analysis from
+ * `DataFlowImplSpecific::Public` and adds a global analysis, mainly exposed
+ * through the `Make` and `MakeWithState` modules.
+ */
+
+private import DataFlowImplCommon
+private import DataFlowImplSpecific::Private
+import DataFlowImplSpecific::Public
+import DataFlowImplCommonPublic
+private import DataFlowImpl
+
+/** An input configuration for data flow. */
+signature module ConfigSig {
+  /**
+   * Holds if `source` is a relevant data flow source.
+   */
+  predicate isSource(Node source);
+
+  /**
+   * Holds if `sink` is a relevant data flow sink.
+   */
+  predicate isSink(Node sink);
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  default predicate isBarrier(Node node) { none() }
+
+  /** Holds if data flow into `node` is prohibited. */
+  default predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  default predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  default predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  default predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  default int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  default FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  default predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  default predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (as it is in a `path-problem` query).
+   */
+  default predicate includeHiddenNodes() { none() }
+}
+
+/** An input configuration for data flow using flow state. */
+signature module StateConfigSig {
+  bindingset[this]
+  class FlowState;
+
+  /**
+   * Holds if `source` is a relevant data flow source with the given initial
+   * `state`.
+   */
+  predicate isSource(Node source, FlowState state);
+
+  /**
+   * Holds if `sink` is a relevant data flow sink accepting `state`.
+   */
+  predicate isSink(Node sink, FlowState state);
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  default predicate isBarrier(Node node) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited when the flow state is
+   * `state`.
+   */
+  predicate isBarrier(Node node, FlowState state);
+
+  /** Holds if data flow into `node` is prohibited. */
+  default predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  default predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  default predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
+   */
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2);
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  default predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  default int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  default FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  default predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  default predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (as it is in a `path-problem` query).
+   */
+  default predicate includeHiddenNodes() { none() }
+}
+
+/**
+ * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
+ * measured in approximate number of interprocedural steps.
+ */
+signature int explorationLimitSig();
+
+/**
+ * The output of a data flow computation.
+ */
+signature module DataFlowSig {
+  /**
+   * A `Node` augmented with a call context (except for sinks) and an access path.
+   * Only those `PathNode`s that are reachable from a source, and which can reach a sink, are generated.
+   */
+  class PathNode;
+
+  /**
+   * Holds if data can flow from `source` to `sink`.
+   *
+   * The corresponding paths are generated from the end-points and the graph
+   * included in the module `PathGraph`.
+   */
+  predicate hasFlowPath(PathNode source, PathNode sink);
+
+  /**
+   * Holds if data can flow from `source` to `sink`.
+   */
+  predicate hasFlow(Node source, Node sink);
+
+  /**
+   * Holds if data can flow from some source to `sink`.
+   */
+  predicate hasFlowTo(Node sink);
+
+  /**
+   * Holds if data can flow from some source to `sink`.
+   */
+  predicate hasFlowToExpr(DataFlowExpr sink);
+}
+
+/**
+ * Constructs a standard data flow computation.
+ */
+module Make<ConfigSig Config> implements DataFlowSig {
+  private module C implements FullStateConfigSig {
+    import DefaultState<Config>
+    import Config
+  }
+
+  import Impl<C>
+}
+
+/**
+ * Constructs a data flow computation using flow state.
+ */
+module MakeWithState<StateConfigSig Config> implements DataFlowSig {
+  private module C implements FullStateConfigSig {
+    import Config
+  }
+
+  import Impl<C>
+}

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl1.qll

@@ -0,0 +1,396 @@
+/**
+ * DEPRECATED: Use `Make` and `MakeWithState` instead.
+ *
+ * Provides a `Configuration` class backwards-compatible interface to the data
+ * flow library.
+ */
+
+private import DataFlowImplCommon
+private import DataFlowImplSpecific::Private
+import DataFlowImplSpecific::Public
+private import DataFlowImpl
+import DataFlowImplCommonPublic
+import FlowStateString
+
+/**
+ * A configuration of interprocedural data flow analysis. This defines
+ * sources, sinks, and any other configurable aspect of the analysis. Each
+ * use of the global data flow library must define its own unique extension
+ * of this abstract class. To create a configuration, extend this class with
+ * a subclass whose characteristic predicate is a unique singleton string.
+ * For example, write
+ *
+ * ```ql
+ * class MyAnalysisConfiguration extends DataFlow::Configuration {
+ *   MyAnalysisConfiguration() { this = "MyAnalysisConfiguration" }
+ *   // Override `isSource` and `isSink`.
+ *   // Optionally override `isBarrier`.
+ *   // Optionally override `isAdditionalFlowStep`.
+ * }
+ * ```
+ * Conceptually, this defines a graph where the nodes are `DataFlow::Node`s and
+ * the edges are those data-flow steps that preserve the value of the node
+ * along with any additional edges defined by `isAdditionalFlowStep`.
+ * Specifying nodes in `isBarrier` will remove those nodes from the graph, and
+ * specifying nodes in `isBarrierIn` and/or `isBarrierOut` will remove in-going
+ * and/or out-going edges from those nodes, respectively.
+ *
+ * Then, to query whether there is flow between some `source` and `sink`,
+ * write
+ *
+ * ```ql
+ * exists(MyAnalysisConfiguration cfg | cfg.hasFlow(source, sink))
+ * ```
+ *
+ * Multiple configurations can coexist, but two classes extending
+ * `DataFlow::Configuration` should never depend on each other. One of them
+ * should instead depend on a `DataFlow2::Configuration`, a
+ * `DataFlow3::Configuration`, or a `DataFlow4::Configuration`.
+ */
+abstract class Configuration extends string {
+  bindingset[this]
+  Configuration() { any() }
+
+  /**
+   * Holds if `source` is a relevant data flow source.
+   */
+  predicate isSource(Node source) { none() }
+
+  /**
+   * Holds if `source` is a relevant data flow source with the given initial
+   * `state`.
+   */
+  predicate isSource(Node source, FlowState state) { none() }
+
+  /**
+   * Holds if `sink` is a relevant data flow sink.
+   */
+  predicate isSink(Node sink) { none() }
+
+  /**
+   * Holds if `sink` is a relevant data flow sink accepting `state`.
+   */
+  predicate isSink(Node sink, FlowState state) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited. This completely removes
+   * `node` from the data flow graph.
+   */
+  predicate isBarrier(Node node) { none() }
+
+  /**
+   * Holds if data flow through `node` is prohibited when the flow state is
+   * `state`.
+   */
+  predicate isBarrier(Node node, FlowState state) { none() }
+
+  /** Holds if data flow into `node` is prohibited. */
+  predicate isBarrierIn(Node node) { none() }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  predicate isBarrierOut(Node node) { none() }
+
+  /**
+   * DEPRECATED: Use `isBarrier` and `BarrierGuard` module instead.
+   *
+   * Holds if data flow through nodes guarded by `guard` is prohibited.
+   */
+  deprecated predicate isBarrierGuard(BarrierGuard guard) { none() }
+
+  /**
+   * DEPRECATED: Use `isBarrier` and `BarrierGuard` module instead.
+   *
+   * Holds if data flow through nodes guarded by `guard` is prohibited when
+   * the flow state is `state`
+   */
+  deprecated predicate isBarrierGuard(BarrierGuard guard, FlowState state) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   */
+  predicate isAdditionalFlowStep(Node node1, Node node2) { none() }
+
+  /**
+   * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps.
+   * This step is only applicable in `state1` and updates the flow state to `state2`.
+   */
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
+    none()
+  }
+
+  /**
+   * Holds if an arbitrary number of implicit read steps of content `c` may be
+   * taken at `node`.
+   */
+  predicate allowImplicitRead(Node node, ContentSet c) { none() }
+
+  /**
+   * Gets the virtual dispatch branching limit when calculating field flow.
+   * This can be overridden to a smaller value to improve performance (a
+   * value of 0 disables field flow), or a larger value to get more results.
+   */
+  int fieldFlowBranchLimit() { result = 2 }
+
+  /**
+   * Gets a data flow configuration feature to add restrictions to the set of
+   * valid flow paths.
+   *
+   * - `FeatureHasSourceCallContext`:
+   *    Assume that sources have some existing call context to disallow
+   *    conflicting return-flow directly following the source.
+   * - `FeatureHasSinkCallContext`:
+   *    Assume that sinks have some existing call context to disallow
+   *    conflicting argument-to-parameter flow directly preceding the sink.
+   * - `FeatureEqualSourceSinkCallContext`:
+   *    Implies both of the above and additionally ensures that the entire flow
+   *    path preserves the call context.
+   *
+   * These features are generally not relevant for typical end-to-end data flow
+   * queries, but should only be used for constructing paths that need to
+   * somehow be pluggable in another path context.
+   */
+  FlowFeature getAFeature() { none() }
+
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
+  /**
+   * Holds if data may flow from `source` to `sink` for this configuration.
+   */
+  predicate hasFlow(Node source, Node sink) { hasFlow(source, sink, this) }
+
+  /**
+   * Holds if data may flow from `source` to `sink` for this configuration.
+   *
+   * The corresponding paths are generated from the end-points and the graph
+   * included in the module `PathGraph`.
+   */
+  predicate hasFlowPath(PathNode source, PathNode sink) { hasFlowPath(source, sink, this) }
+
+  /**
+   * Holds if data may flow from some source to `sink` for this configuration.
+   */
+  predicate hasFlowTo(Node sink) { hasFlowTo(sink, this) }
+
+  /**
+   * Holds if data may flow from some source to `sink` for this configuration.
+   */
+  predicate hasFlowToExpr(DataFlowExpr sink) { this.hasFlowTo(exprNode(sink)) }
+
+  /**
+   * DEPRECATED: Use `FlowExploration<explorationLimit>` instead.
+   *
+   * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
+   * measured in approximate number of interprocedural steps.
+   */
+  deprecated int explorationLimit() { none() }
+
+  /**
+   * Holds if hidden nodes should be included in the data flow graph.
+   *
+   * This feature should only be used for debugging or when the data flow graph
+   * is not visualized (for example in a `path-problem` query).
+   */
+  predicate includeHiddenNodes() { none() }
+}
+
+/**
+ * This class exists to prevent mutual recursion between the user-overridden
+ * member predicates of `Configuration` and the rest of the data-flow library.
+ * Good performance cannot be guaranteed in the presence of such recursion, so
+ * it should be replaced by using more than one copy of the data flow library.
+ */
+abstract private class ConfigurationRecursionPrevention extends Configuration {
+  bindingset[this]
+  ConfigurationRecursionPrevention() { any() }
+
+  override predicate hasFlow(Node source, Node sink) {
+    strictcount(Node n | this.isSource(n)) < 0
+    or
+    strictcount(Node n | this.isSource(n, _)) < 0
+    or
+    strictcount(Node n | this.isSink(n)) < 0
+    or
+    strictcount(Node n | this.isSink(n, _)) < 0
+    or
+    strictcount(Node n1, Node n2 | this.isAdditionalFlowStep(n1, n2)) < 0
+    or
+    strictcount(Node n1, Node n2 | this.isAdditionalFlowStep(n1, _, n2, _)) < 0
+    or
+    super.hasFlow(source, sink)
+  }
+}
+
+/** A bridge class to access the deprecated `isBarrierGuard`. */
+private class BarrierGuardGuardedNodeBridge extends Unit {
+  abstract predicate guardedNode(Node n, Configuration config);
+
+  abstract predicate guardedNode(Node n, FlowState state, Configuration config);
+}
+
+private class BarrierGuardGuardedNode extends BarrierGuardGuardedNodeBridge {
+  deprecated override predicate guardedNode(Node n, Configuration config) {
+    exists(BarrierGuard g |
+      config.isBarrierGuard(g) and
+      n = g.getAGuardedNode()
+    )
+  }
+
+  deprecated override predicate guardedNode(Node n, FlowState state, Configuration config) {
+    exists(BarrierGuard g |
+      config.isBarrierGuard(g, state) and
+      n = g.getAGuardedNode()
+    )
+  }
+}
+
+private FlowState relevantState(Configuration config) {
+  config.isSource(_, result) or
+  config.isSink(_, result) or
+  config.isBarrier(_, result) or
+  config.isAdditionalFlowStep(_, result, _, _) or
+  config.isAdditionalFlowStep(_, _, _, result)
+}
+
+private newtype TConfigState =
+  TMkConfigState(Configuration config, FlowState state) {
+    state = relevantState(config) or state instanceof FlowStateEmpty
+  }
+
+private Configuration getConfig(TConfigState state) { state = TMkConfigState(result, _) }
+
+private FlowState getState(TConfigState state) { state = TMkConfigState(_, result) }
+
+private predicate singleConfiguration() { 1 = strictcount(Configuration c) }
+
+private module Config implements FullStateConfigSig {
+  class FlowState = TConfigState;
+
+  predicate isSource(Node source, FlowState state) {
+    getConfig(state).isSource(source, getState(state))
+    or
+    getConfig(state).isSource(source) and getState(state) instanceof FlowStateEmpty
+  }
+
+  predicate isSink(Node sink, FlowState state) {
+    getConfig(state).isSink(sink, getState(state))
+    or
+    getConfig(state).isSink(sink) and getState(state) instanceof FlowStateEmpty
+  }
+
+  predicate isBarrier(Node node) { none() }
+
+  predicate isBarrier(Node node, FlowState state) {
+    getConfig(state).isBarrier(node, getState(state)) or
+    getConfig(state).isBarrier(node) or
+    any(BarrierGuardGuardedNodeBridge b).guardedNode(node, getState(state), getConfig(state)) or
+    any(BarrierGuardGuardedNodeBridge b).guardedNode(node, getConfig(state))
+  }
+
+  predicate isBarrierIn(Node node) { any(Configuration config).isBarrierIn(node) }
+
+  predicate isBarrierOut(Node node) { any(Configuration config).isBarrierOut(node) }
+
+  predicate isAdditionalFlowStep(Node node1, Node node2) {
+    singleConfiguration() and
+    any(Configuration config).isAdditionalFlowStep(node1, node2)
+  }
+
+  predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {
+    getConfig(state1).isAdditionalFlowStep(node1, getState(state1), node2, getState(state2)) and
+    getConfig(state2) = getConfig(state1)
+    or
+    not singleConfiguration() and
+    getConfig(state1).isAdditionalFlowStep(node1, node2) and
+    state2 = state1
+  }
+
+  predicate allowImplicitRead(Node node, ContentSet c) {
+    any(Configuration config).allowImplicitRead(node, c)
+  }
+
+  int fieldFlowBranchLimit() { result = min(any(Configuration config).fieldFlowBranchLimit()) }
+
+  FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
+
+  predicate sourceGrouping(Node source, string sourceGroup) {
+    any(Configuration config).sourceGrouping(source, sourceGroup)
+  }
+
+  predicate sinkGrouping(Node sink, string sinkGroup) {
+    any(Configuration config).sinkGrouping(sink, sinkGroup)
+  }
+
+  predicate includeHiddenNodes() { any(Configuration config).includeHiddenNodes() }
+}
+
+private import Impl<Config> as I
+import I
+
+/**
+ * A `Node` augmented with a call context (except for sinks), an access path, and a configuration.
+ * Only those `PathNode`s that are reachable from a source, and which can reach a sink, are generated.
+ */
+class PathNode instanceof I::PathNode {
+  /** Gets a textual representation of this element. */
+  final string toString() { result = super.toString() }
+
+  /**
+   * Gets a textual representation of this element, including a textual
+   * representation of the call context.
+   */
+  final string toStringWithContext() { result = super.toStringWithContext() }
+
+  /**
+   * Holds if this element is at the specified location.
+   * The location spans column `startcolumn` of line `startline` to
+   * column `endcolumn` of line `endline` in file `filepath`.
+   * For more information, see
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
+   */
+  final predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    super.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+  }
+
+  /** Gets the underlying `Node`. */
+  final Node getNode() { result = super.getNode() }
+
+  /** Gets the `FlowState` of this node. */
+  final FlowState getState() { result = getState(super.getState()) }
+
+  /** Gets the associated configuration. */
+  final Configuration getConfiguration() { result = getConfig(super.getState()) }
+
+  /** Gets a successor of this node, if any. */
+  final PathNode getASuccessor() { result = super.getASuccessor() }
+
+  /** Holds if this node is a source. */
+  final predicate isSource() { super.isSource() }
+
+  /** Holds if this node is a grouping of source nodes. */
+  final predicate isSourceGroup(string group) { super.isSourceGroup(group) }
+
+  /** Holds if this node is a grouping of sink nodes. */
+  final predicate isSinkGroup(string group) { super.isSinkGroup(group) }
+}
+
+private predicate hasFlow(Node source, Node sink, Configuration config) {
+  exists(PathNode source0, PathNode sink0 |
+    hasFlowPath(source0, sink0, config) and
+    source0.getNode() = source and
+    sink0.getNode() = sink
+  )
+}
+
+private predicate hasFlowPath(PathNode source, PathNode sink, Configuration config) {
+  hasFlowPath(source, sink) and source.getConfiguration() = config
+}
+
+private predicate hasFlowTo(Node sink, Configuration config) { hasFlow(_, sink, config) }
+
+predicate flowsTo = hasFlow/3;

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll

@@ -3,15 +3,18 @@ private import DataFlowImplSpecific::Public
 import Cached
 
 module DataFlowImplCommonPublic {
-  /** A state value to track during data flow. */
-  class FlowState = string;
+  /** Provides `FlowState = string`. */
+  module FlowStateString {
+    /** A state value to track during data flow. */
+    class FlowState = string;
 
-  /**
-   * The default state, which is used when the state is unspecified for a source
-   * or a sink.
-   */
-  class FlowStateEmpty extends FlowState {
-    FlowStateEmpty() { this = "" }
+    /**
+     * The default state, which is used when the state is unspecified for a source
+     * or a sink.
+     */
+    class FlowStateEmpty extends FlowState {
+      FlowStateEmpty() { this = "" }
+    }
   }
 
   private newtype TFlowFeature =

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll

@@ -414,3 +414,12 @@ private class MyConsistencyConfiguration extends Consistency::ConsistencyConfigu
     any()
   }
 }
+
+/**
+ * Gets an additional term that is added to the `join` and `branch` computations to reflect
+ * an additional forward or backwards branching factor that is not taken into account
+ * when calculating the (virtual) dispatch cost.
+ *
+ * Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter.
+ */
+int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p) { none() }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/PrintIRLocalFlow.qll

@@ -97,23 +97,23 @@ private string getNodeProperty(DataFlow::Node node, string key) {
     |
       kind, ", "
     )
-  or
-  // Is there partial flow from a source to this node?
-  // This property will only be emitted if partial flow is enabled by overriding
-  // `DataFlow::Configuration::explorationLimit()`.
-  key = "pflow" and
-  result =
-    strictconcat(DataFlow::PartialPathNode sourceNode, DataFlow::PartialPathNode destNode, int dist,
-      int order1, int order2 |
-      any(DataFlow::Configuration cfg).hasPartialFlow(sourceNode, destNode, dist) and
-      destNode.getNode() = node and
-      // Only print flow from a source in the same function.
-      sourceNode.getNode().getEnclosingCallable() = node.getEnclosingCallable()
-    |
-      nodeId(sourceNode.getNode(), order1, order2) + "+" + dist.toString(), ", "
-      order by
-        order1, order2, dist desc
-    )
+  // or
+  // // Is there partial flow from a source to this node?
+  // // This property will only be emitted if partial flow is enabled by overriding
+  // // `DataFlow::Configuration::explorationLimit()`.
+  // key = "pflow" and
+  // result =
+  //   strictconcat(DataFlow::PartialPathNode sourceNode, DataFlow::PartialPathNode destNode, int dist,
+  //     int order1, int order2 |
+  //     any(DataFlow::Configuration cfg).hasPartialFlow(sourceNode, destNode, dist) and
+  //     destNode.getNode() = node and
+  //     // Only print flow from a source in the same function.
+  //     sourceNode.getNode().getEnclosingCallable() = node.getEnclosingCallable()
+  //   |
+  //     nodeId(sourceNode.getNode(), order1, order2) + "+" + dist.toString(), ", "
+  //     order by
+  //       order1, order2, dist desc
+  //   )
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTracking.qll

@@ -0,0 +1,63 @@
+/**
+ * Provides classes for performing local (intra-procedural) and
+ * global (inter-procedural) taint-tracking analyses.
+ */
+
+import TaintTrackingParameter::Public
+private import TaintTrackingParameter::Private
+
+private module AddTaintDefaults<DataFlowInternal::FullStateConfigSig Config> implements
+DataFlowInternal::FullStateConfigSig {
+  import Config
+
+  predicate isBarrier(DataFlow::Node node) {
+    Config::isBarrier(node) or defaultTaintSanitizer(node)
+  }
+
+  predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
+    Config::isAdditionalFlowStep(node1, node2) or
+    defaultAdditionalTaintStep(node1, node2)
+  }
+
+  predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c) {
+    Config::allowImplicitRead(node, c)
+    or
+    (
+      Config::isSink(node, _) or
+      Config::isAdditionalFlowStep(node, _) or
+      Config::isAdditionalFlowStep(node, _, _, _)
+    ) and
+    defaultImplicitTaintRead(node, c)
+  }
+}
+
+/**
+ * Constructs a standard taint tracking computation.
+ */
+module Make<DataFlow::ConfigSig Config> implements DataFlow::DataFlowSig {
+  private module Config0 implements DataFlowInternal::FullStateConfigSig {
+    import DataFlowInternal::DefaultState<Config>
+    import Config
+  }
+
+  private module C implements DataFlowInternal::FullStateConfigSig {
+    import AddTaintDefaults<Config0>
+  }
+
+  import DataFlowInternal::Impl<C>
+}
+
+/**
+ * Constructs a taint tracking computation using flow state.
+ */
+module MakeWithState<DataFlow::StateConfigSig Config> implements DataFlow::DataFlowSig {
+  private module Config0 implements DataFlowInternal::FullStateConfigSig {
+    import Config
+  }
+
+  private module C implements DataFlowInternal::FullStateConfigSig {
+    import AddTaintDefaults<Config0>
+  }
+
+  import DataFlowInternal::Impl<C>
+}

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingParameter.qll

@@ -2,4 +2,5 @@ import semmle.code.cpp.ir.dataflow.internal.TaintTrackingUtil as Public
 
 module Private {
   import semmle.code.cpp.ir.dataflow.DataFlow::DataFlow as DataFlow
+  import semmle.code.cpp.ir.dataflow.internal.DataFlowImpl as DataFlowInternal
 }

cpp/ql/src/Security/CWE/CWE-078/ExecTainted.ql

@@ -22,7 +22,7 @@ import semmle.code.cpp.ir.dataflow.TaintTracking
 import semmle.code.cpp.ir.dataflow.TaintTracking2
 import semmle.code.cpp.security.FlowSources
 import semmle.code.cpp.models.implementations.Strcat
-import DataFlow::PathGraph
+import ExecTaint::PathGraph
 
 Expr sinkAsArgumentIndirection(DataFlow::Node sink) {
   result =
@@ -67,28 +67,28 @@ predicate interestingConcatenation(DataFlow::Node fst, DataFlow::Node snd) {
   )
 }
 
-class ConcatState extends DataFlow::FlowState {
-  ConcatState() { this = "ConcatState" }
+newtype TState =
+  TConcatState() or
+  TExecState(DataFlow::Node fst, DataFlow::Node snd) { interestingConcatenation(fst, snd) }
+
+class ConcatState extends TConcatState {
+  string toString() { result = "ConcatState" }
 }
 
-class ExecState extends DataFlow::FlowState {
+class ExecState extends TExecState {
   DataFlow::Node fst;
   DataFlow::Node snd;
 
-  ExecState() {
-    this =
-      "ExecState (" + fst.getLocation() + " | " + fst + ", " + snd.getLocation() + " | " + snd + ")" and
-    interestingConcatenation(pragma[only_bind_into](fst), pragma[only_bind_into](snd))
-  }
+  ExecState() { this = TExecState(fst, snd) }
 
   DataFlow::Node getFstNode() { result = fst }
 
   DataFlow::Node getSndNode() { result = snd }
 
   /** Holds if this is a possible `ExecState` for `sink`. */
-  predicate isFeasibleForSink(DataFlow::Node sink) {
-    any(ExecStateConfiguration conf).hasFlow(snd, sink)
-  }
+  predicate isFeasibleForSink(DataFlow::Node sink) { ExecState::hasFlow(snd, sink) }
+
+  string toString() { result = "ExecState" }
 }
 
 /**
@@ -96,45 +96,42 @@ class ExecState extends DataFlow::FlowState {
  * given sink. This avoids a cartesian product between all sinks and all `ExecState`s in
  * `ExecTaintConfiguration::isSink`.
  */
-class ExecStateConfiguration extends TaintTracking2::Configuration {
-  ExecStateConfiguration() { this = "ExecStateConfiguration" }
-
-  override predicate isSource(DataFlow::Node source) {
+module ExecStateConfiguration implements DataFlow::ConfigSig {
+  predicate isSource(DataFlow::Node source) {
     exists(ExecState state | state.getSndNode() = source)
   }
 
-  override predicate isSink(DataFlow::Node sink) {
-    shellCommand(sinkAsArgumentIndirection(sink), _)
-  }
+  predicate isSink(DataFlow::Node sink) { shellCommand(sinkAsArgumentIndirection(sink), _) }
 
-  override predicate isSanitizerOut(DataFlow::Node node) {
-    isSink(node, _) // Prevent duplicates along a call chain, since `shellCommand` will include wrappers
+  predicate isBarrierOut(DataFlow::Node node) {
+    isSink(node) // Prevent duplicates along a call chain, since `shellCommand` will include wrappers
   }
 }
 
-class ExecTaintConfiguration extends TaintTracking::Configuration {
-  ExecTaintConfiguration() { this = "ExecTaintConfiguration" }
+module ExecState = TaintTracking::Make<ExecStateConfiguration>;
 
-  override predicate isSource(DataFlow::Node source, DataFlow::FlowState state) {
+module ExecTaintConfiguration implements DataFlow::StateConfigSig {
+  class FlowState = TState;
+
+  predicate isSource(DataFlow::Node source, FlowState state) {
     source instanceof FlowSource and
     state instanceof ConcatState
   }
 
-  override predicate isSink(DataFlow::Node sink, DataFlow::FlowState state) {
-    any(ExecStateConfiguration conf).isSink(sink) and
+  predicate isSink(DataFlow::Node sink, FlowState state) {
+    ExecStateConfiguration::isSink(sink) and
     state.(ExecState).isFeasibleForSink(sink)
   }
 
-  override predicate isAdditionalTaintStep(
-    DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,
-    DataFlow::FlowState state2
+  predicate isAdditionalFlowStep(
+    DataFlow::Node node1, FlowState state1, DataFlow::Node node2, FlowState state2
   ) {
     state1 instanceof ConcatState and
     state2.(ExecState).getFstNode() = node1 and
     state2.(ExecState).getSndNode() = node2
   }
 
-  override predicate isSanitizer(DataFlow::Node node, DataFlow::FlowState state) {
+  predicate isBarrier(DataFlow::Node node, FlowState state) {
     (
       node.asInstruction().getResultType() instanceof IntegralType
       or
@@ -143,16 +140,18 @@ class ExecTaintConfiguration extends TaintTracking::Configuration {
     state instanceof ConcatState
   }
 
-  override predicate isSanitizerOut(DataFlow::Node node) {
+  predicate isBarrierOut(DataFlow::Node node) {
     isSink(node, _) // Prevent duplicates along a call chain, since `shellCommand` will include wrappers
   }
 }
 
+module ExecTaint = TaintTracking::MakeWithState<ExecTaintConfiguration>;
+
 from
-  ExecTaintConfiguration conf, DataFlow::PathNode sourceNode, DataFlow::PathNode sinkNode,
-  string taintCause, string callChain, DataFlow::Node concatResult
+  ExecTaint::PathNode sourceNode, ExecTaint::PathNode sinkNode, string taintCause, string callChain,
+  DataFlow::Node concatResult
 where
-  conf.hasFlowPath(sourceNode, sinkNode) and
+  ExecTaint::hasFlowPath(sourceNode, sinkNode) and
   taintCause = sourceNode.getNode().(FlowSource).getSourceType() and
   shellCommand(sinkAsArgumentIndirection(sinkNode.getNode()), callChain) and
   concatResult = sinkNode.getState().(ExecState).getSndNode()

cpp/ql/src/Security/CWE/CWE-732/DoNotCreateWorldWritable.c

@@ -1,11 +1,11 @@
-int write_default_config_bad() {
+void write_default_config_bad() {
 	// BAD - this is world-writable so any user can overwrite the config
-	FILE* out = creat(OUTFILE, 0666);
-	fprintf(out, DEFAULT_CONFIG);
+	int out = creat(OUTFILE, 0666);
+	dprintf(out, DEFAULT_CONFIG);
 }
 
-int write_default_config_good() {
+void write_default_config_good() {
 	// GOOD - this allows only the current user to modify the file
-	FILE* out = creat(OUTFILE, S_IWUSR | S_IRUSR);
-	fprintf(out, DEFAULT_CONFIG);
+	int out = creat(OUTFILE, S_IWUSR | S_IRUSR);
+	dprintf(out, DEFAULT_CONFIG);
 }

cpp/ql/src/experimental/Security/CWE/CWE-369/DivideByZeroUsingReturnValue.cpp

@@ -0,0 +1,7 @@
+
+...
+  a = getc(f);
+  if (a < 123) ret = 123/a; // BAD
+...
+  if (a != 0) ret = 123/a; // GOOD
+...

cpp/ql/src/experimental/Security/CWE/CWE-369/DivideByZeroUsingReturnValue.qhelp

@@ -0,0 +1,23 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+<overview>
+<p> Possible cases of division by zero when using the return value from functions.</p>
+
+</overview>
+
+<example>
+<p>The following example shows the use of a function with an error when using the return value and without an error.</p>
+<sample src="DivideByZeroUsingReturnValue.cpp" />
+
+</example>
+<references>
+
+<li>
+  CERT Coding Standard:
+  <a href="https://wiki.sei.cmu.edu/confluence/display/c/INT33-C.+Ensure+that+division+and+remainder+operations+do+not+result+in+divide-by-zero+errors">INT33-C. Ensure that division and remainder operations do not result in divide-by-zero errors - SEI CERT C Coding Standard - Confluence</a>.
+</li>
+
+</references>
+</qhelp>

cpp/ql/src/experimental/Security/CWE/CWE-369/DivideByZeroUsingReturnValue.ql

@@ -0,0 +1,274 @@
+/**
+ * @name Divide by zero using return value
+ * @description Possible cases of division by zero when using the return value from functions.
+ * @kind problem
+ * @id cpp/divide-by-zero-using-return-value
+ * @problem.severity warning
+ * @precision medium
+ * @tags correctness
+ *       security
+ *       external/cwe/cwe-369
+ */
+
+import cpp
+import semmle.code.cpp.valuenumbering.GlobalValueNumbering
+import semmle.code.cpp.controlflow.Guards
+
+/** Holds if function `fn` can return a value equal to value `val` */
+predicate mayBeReturnValue(Function fn, float val) {
+  exists(Expr tmpExp, ReturnStmt rs |
+    tmpExp.getValue().toFloat() = val and
+    rs.getEnclosingFunction() = fn and
+    (
+      globalValueNumber(rs.getExpr()) = globalValueNumber(tmpExp)
+      or
+      exists(AssignExpr ae |
+        ae.getLValue().(VariableAccess).getTarget() =
+          globalValueNumber(rs.getExpr()).getAnExpr().(VariableAccess).getTarget() and
+        globalValueNumber(ae.getRValue()) = globalValueNumber(tmpExp)
+      )
+      or
+      exists(Initializer it |
+        globalValueNumber(it.getExpr()) = globalValueNumber(tmpExp) and
+        it.getDeclaration().(Variable).getAnAccess().getTarget() =
+          globalValueNumber(rs.getExpr()).getAnExpr().(VariableAccess).getTarget()
+      )
+    )
+  )
+}
+
+/** Holds if function `fn` can return a value equal zero */
+predicate mayBeReturnZero(Function fn) {
+  mayBeReturnValue(fn, 0)
+  or
+  fn.hasName([
+      "iswalpha", "iswlower", "iswprint", "iswspace", "iswblank", "iswupper", "iswcntrl",
+      "iswctype", "iswalnum", "iswgraph", "iswxdigit", "iswdigit", "iswpunct", "isblank", "isupper",
+      "isgraph", "isalnum", "ispunct", "islower", "isspace", "isprint", "isxdigit", "iscntrl",
+      "isdigit", "isalpha", "timespec_get", "feof", "atomic_is_lock_free",
+      "atomic_compare_exchange", "thrd_equal", "isfinite", "islessequal", "isnan", "isgreater",
+      "signbit", "isinf", "islessgreater", "isnormal", "isless", "isgreaterequal", "isunordered",
+      "ferror"
+    ])
+  or
+  fn.hasName([
+      "thrd_sleep", "feenv", "feholdexcept", "feclearexcept", "feexceptflag", "feupdateenv",
+      "remove", "fflush", "setvbuf", "fgetpos", "fsetpos", "fclose", "rename", "fseek", "raise"
+    ])
+  or
+  fn.hasName(["tss_get", "gets"])
+  or
+  fn.hasName(["getc", "atoi"])
+}
+
+/** Gets the Guard which compares the expression `bound` */
+pragma[inline]
+GuardCondition checkByValue(Expr bound, Expr val) {
+  exists(GuardCondition gc |
+    (
+      gc.ensuresEq(bound, val, _, _, _) or
+      gc.ensuresEq(val, bound, _, _, _) or
+      gc.ensuresLt(bound, val, _, _, _) or
+      gc.ensuresLt(val, bound, _, _, _) or
+      gc = globalValueNumber(bound).getAnExpr()
+    ) and
+    result = gc
+  )
+}
+
+/** Holds if there are no comparisons between the value returned by possible function calls `compArg` and the value `valArg`, or when these comparisons do not exclude equality to the value `valArg`. */
+pragma[inline]
+predicate compareFunctionWithValue(Expr guardExp, Function compArg, Expr valArg) {
+  not exists(Expr exp |
+    exp.getAChild*() = globalValueNumber(compArg.getACallToThisFunction()).getAnExpr() and
+    checkByValue(exp, valArg).controls(guardExp.getBasicBlock(), _)
+  )
+  or
+  exists(GuardCondition gc |
+    (
+      gc.ensuresEq(globalValueNumber(compArg.getACallToThisFunction()).getAnExpr(), valArg, 0,
+        guardExp.getBasicBlock(), true)
+      or
+      gc.ensuresEq(valArg, globalValueNumber(compArg.getACallToThisFunction()).getAnExpr(), 0,
+        guardExp.getBasicBlock(), true)
+      or
+      gc.ensuresLt(globalValueNumber(compArg.getACallToThisFunction()).getAnExpr(), valArg, 0,
+        guardExp.getBasicBlock(), false)
+      or
+      gc.ensuresLt(valArg, globalValueNumber(compArg.getACallToThisFunction()).getAnExpr(), 0,
+        guardExp.getBasicBlock(), false)
+    )
+    or
+    exists(Expr exp |
+      exp.getValue().toFloat() > valArg.getValue().toFloat() and
+      gc.ensuresLt(globalValueNumber(compArg.getACallToThisFunction()).getAnExpr(), exp, 0,
+        guardExp.getBasicBlock(), true)
+      or
+      exp.getValue().toFloat() < valArg.getValue().toFloat() and
+      gc.ensuresLt(exp, globalValueNumber(compArg.getACallToThisFunction()).getAnExpr(), 0,
+        guardExp.getBasicBlock(), true)
+    )
+  )
+  or
+  valArg.getValue().toFloat() = 0 and
+  exists(NotExpr ne, IfStmt ifne |
+    ne.getOperand() = globalValueNumber(compArg.getACallToThisFunction()).getAnExpr() and
+    ifne.getCondition() = ne and
+    ifne.getThen().getAChild*() = guardExp
+  )
+}
+
+/** Wraping predicate for call `compareFunctionWithValue`. */
+pragma[inline]
+predicate checkConditions1(Expr div, Function fn, float changeInt) {
+  exists(Expr val |
+    val.getEnclosingFunction() = fn and
+    val.getValue().toFloat() = changeInt and
+    compareFunctionWithValue(div, fn, val)
+  )
+}
+
+/** Holds if there are no comparisons between the value `compArg` and the value `valArg`, or when these comparisons do not exclude equality to the value `valArg`. */
+pragma[inline]
+predicate compareExprWithValue(Expr guardExp, Expr compArg, Expr valArg) {
+  not exists(Expr exp |
+    exp.getAChild*() = globalValueNumber(compArg).getAnExpr() and
+    checkByValue(exp, valArg).controls(guardExp.getBasicBlock(), _)
+  )
+  or
+  exists(GuardCondition gc |
+    (
+      gc.ensuresEq(globalValueNumber(compArg).getAnExpr(), valArg, 0, guardExp.getBasicBlock(), true)
+      or
+      gc.ensuresEq(valArg, globalValueNumber(compArg).getAnExpr(), 0, guardExp.getBasicBlock(), true)
+      or
+      gc.ensuresLt(globalValueNumber(compArg).getAnExpr(), valArg, 0, guardExp.getBasicBlock(),
+        false)
+      or
+      gc.ensuresLt(valArg, globalValueNumber(compArg).getAnExpr(), 0, guardExp.getBasicBlock(),
+        false)
+    )
+    or
+    exists(Expr exp |
+      exp.getValue().toFloat() > valArg.getValue().toFloat() and
+      gc.ensuresLt(globalValueNumber(compArg).getAnExpr(), exp, 0, guardExp.getBasicBlock(), true)
+      or
+      exp.getValue().toFloat() < valArg.getValue().toFloat() and
+      gc.ensuresLt(exp, globalValueNumber(compArg).getAnExpr(), 0, guardExp.getBasicBlock(), true)
+    )
+  )
+  or
+  valArg.getValue().toFloat() = 0 and
+  exists(NotExpr ne, IfStmt ifne |
+    ne.getOperand() = globalValueNumber(compArg).getAnExpr() and
+    ifne.getCondition() = ne and
+    ifne.getThen().getAChild*() = guardExp
+  )
+}
+
+/** Wraping predicate for call `compareExprWithValue`. */
+pragma[inline]
+predicate checkConditions2(Expr div, Expr divVal, float changeInt2) {
+  exists(Expr val |
+    (
+      val.getEnclosingFunction() =
+        div.getEnclosingFunction().getACallToThisFunction().getEnclosingFunction() or
+      val.getEnclosingFunction() = div.getEnclosingFunction()
+    ) and
+    val.getValue().toFloat() = changeInt2 and
+    compareExprWithValue(div, divVal, val)
+  )
+}
+
+/** Gets the value of the difference or summand from the expression `src`. */
+float getValueOperand(Expr src, Expr e1, Expr e2) {
+  src.(SubExpr).hasOperands(e1, e2) and
+  result = e2.getValue().toFloat()
+  or
+  src.(AddExpr).hasOperands(e1, e2) and
+  result = -e2.getValue().toFloat()
+}
+
+/** Function the return of the expression `e1` and the multiplication operands, or the left operand of division if `e1` contains a multiplication or division, respectively. */
+Expr getMulDivOperand(Expr e1) {
+  result = e1 or
+  result = e1.(MulExpr).getAnOperand() or
+  result = e1.(DivExpr).getLeftOperand()
+}
+
+/** The class that defines possible variants of the division expression or the search for the remainder. */
+class MyDiv extends Expr {
+  MyDiv() {
+    this instanceof DivExpr or
+    this instanceof RemExpr or
+    this instanceof AssignDivExpr or
+    this instanceof AssignRemExpr
+  }
+
+  Expr getRV() {
+    result = this.(AssignArithmeticOperation).getRValue() or
+    result = this.(BinaryArithmeticOperation).getRightOperand()
+  }
+}
+
+from Expr exp, string msg, Function fn, GVN findVal, float changeInt, MyDiv div
+where
+  findVal = globalValueNumber(fn.getACallToThisFunction()) and
+  (
+    // Look for divide-by-zero operations possible due to the return value of the function `fn`.
+    checkConditions1(div, fn, changeInt) and
+    (
+      // Function return value can be zero.
+      mayBeReturnZero(fn) and
+      getMulDivOperand(globalValueNumber(div.getRV()).getAnExpr()) = findVal.getAnExpr() and
+      changeInt = 0
+      or
+      // Denominator can be sum or difference.
+      changeInt = getValueOperand(div.getRV(), findVal.getAnExpr(), _) and
+      mayBeReturnValue(fn, changeInt)
+    ) and
+    exp = div and
+    msg =
+      "Can lead to division by 0, since the function " + fn.getName() + " can return a value " +
+        changeInt.toString() + "."
+    or
+    // Search for situations where division by zero is possible inside the `divFn` function if the passed argument can be equal to a certain value.
+    exists(int posArg, Expr divVal, FunctionCall divFc, float changeInt2 |
+      // Division is associated with the function argument.
+      exists(Function divFn |
+        divFn.getParameter(posArg).getAnAccess() = divVal and
+        divVal.getEnclosingStmt() = div.getEnclosingStmt() and
+        divFc = divFn.getACallToThisFunction()
+      ) and
+      (
+        divVal = div.getRV() and
+        divFc.getArgument(posArg) != findVal.getAnExpr() and
+        (
+          // Function return value can be zero.
+          mayBeReturnZero(fn) and
+          getMulDivOperand(globalValueNumber(divFc.getArgument(posArg)).getAnExpr()) =
+            findVal.getAnExpr() and
+          changeInt = 0 and
+          changeInt2 = 0
+          or
+          // Denominator can be sum or difference.
+          changeInt = getValueOperand(divFc.getArgument(posArg), findVal.getAnExpr(), _) and
+          mayBeReturnValue(fn, changeInt) and
+          changeInt2 = 0
+        )
+        or
+        // Look for a situation where the difference or subtraction is considered as an argument, and it can be used in the same way.
+        changeInt = getValueOperand(div.getRV(), divVal, _) and
+        changeInt2 = changeInt and
+        mayBeReturnValue(fn, changeInt) and
+        divFc.getArgument(posArg) = findVal.getAnExpr()
+      ) and
+      checkConditions2(div, divVal, changeInt2) and
+      checkConditions1(divFc, fn, changeInt) and
+      exp = divFc and
+      msg =
+        "Can lead to division by 0, since the function " + fn.getName() + " can return a value " +
+          changeInt.toString() + "."
+    )
+  )
+select exp, msg

cpp/ql/src/experimental/Security/CWE/CWE-415/DoubleFree.ql

@@ -14,8 +14,8 @@ import cpp
 
 from FunctionCall fc, FunctionCall fc2, LocalScopeVariable v
 where
-  freeCall(fc, v.getAnAccess()) and
-  freeCall(fc2, v.getAnAccess()) and
+  fc.(DeallocationExpr).getFreedExpr() = v.getAnAccess() and
+  fc2.(DeallocationExpr).getFreedExpr() = v.getAnAccess() and
   fc != fc2 and
   fc.getASuccessor*() = fc2 and
   not exists(Expr exptmp |

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.5.3
+version: 0.5.4-dev
 groups: 
   - cpp
   - queries

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-369/semmle/tests/DivideByZeroUsingReturnValue.expected

@@ -0,0 +1,27 @@
+| test.cpp:47:24:47:31 | ... / ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:48:15:48:34 | ... / ... | Can lead to division by 0, since the function getSize2 can return a value 0. |
+| test.cpp:53:10:53:17 | ... / ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:65:15:65:22 | ... / ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:68:15:68:22 | ... / ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:71:9:71:16 | ... / ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:74:9:74:16 | ... / ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:77:21:77:28 | ... / ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:79:25:79:32 | ... / ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:81:24:81:31 | ... / ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:128:10:128:16 | ... / ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:135:10:135:16 | ... / ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:141:10:141:23 | ... / ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:153:12:153:19 | ... / ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:172:3:172:12 | ... /= ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:173:3:173:12 | ... %= ... | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:187:10:187:17 | ... / ... | Can lead to division by 0, since the function getSizeFloat can return a value 0. |
+| test.cpp:199:12:199:25 | ... / ... | Can lead to division by 0, since the function getSize can return a value -1. |
+| test.cpp:202:12:202:25 | ... / ... | Can lead to division by 0, since the function getSize can return a value 1. |
+| test.cpp:205:10:205:23 | ... / ... | Can lead to division by 0, since the function getSize can return a value 1. |
+| test.cpp:210:10:210:23 | ... / ... | Can lead to division by 0, since the function getSize can return a value 3. |
+| test.cpp:258:3:258:10 | call to badMyDiv | Can lead to division by 0, since the function getSize can return a value 0. |
+| test.cpp:259:3:259:10 | call to badMyDiv | Can lead to division by 0, since the function getSize can return a value 2. |
+| test.cpp:260:3:260:13 | call to badMySubDiv | Can lead to division by 0, since the function getSize can return a value 3. |
+| test.cpp:263:5:263:15 | call to badMySubDiv | Can lead to division by 0, since the function getSize can return a value 3. |
+| test.cpp:273:5:273:12 | call to badMyDiv | Can lead to division by 0, since the function getSize can return a value 3. |
+| test.cpp:275:5:275:12 | call to badMyDiv | Can lead to division by 0, since the function getSize can return a value -1. |

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-369/semmle/tests/DivideByZeroUsingReturnValue.qlref

@@ -0,0 +1 @@
+experimental/Security/CWE/CWE-369/DivideByZeroUsingReturnValue.ql

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-369/semmle/tests/test.cpp

@@ -0,0 +1,278 @@
+typedef struct {}
+FILE;
+int getc(FILE * stream);
+
+int getSize(int type) {
+  int st;
+  switch (type) {
+  case 1:
+    st = 1;
+    break;
+  case 2:
+    st = 2;
+    break;
+  case 3:
+    st = 3;
+    break;
+  case 4:
+    st = -1;
+    break;
+  default:
+    st = 0;
+    break;
+  }
+  return st;
+}
+int getSize2(int type) {
+  int st = 0;
+  switch (type) {
+  case 1:
+    st = 1;
+    break;
+  case 2:
+    st = 2;
+    break;
+  case 3:
+    st = 3;
+    break;
+  case 4:
+    st = -1;
+    break;
+  }
+  return st;
+}
+
+int badTestf1(int type, int met) {
+  int is = getSize(type);
+  if (met == 1) return 123 / is; // BAD
+  else return 123 / getSize2(type); // BAD
+}
+int badTestf2(int type) {
+  int is;
+  is = getSize(type);
+  return 123 / is; // BAD
+}
+
+int badTestf3(int type, int met) {
+  int is;
+  is = getSize(type);
+  switch (met) {
+  case 1:
+    if (is >= 0) return 123 / is; // BAD [NOT DETECTED]
+  case 2:
+    if (0 == is) return 123 / is; // BAD [NOT DETECTED]
+  case 3:
+    if (!is & 123 / is) // BAD
+      return 123;
+  case 4:
+    if (!is | 123 / is) // BAD
+      return 123;
+  case 5:
+    if (123 / is || !is) // BAD 
+      return 123;
+  case 6:
+    if (123 / is && !is) // BAD
+      return 123;
+  case 7:
+    if (!is) return 123 / is; // BAD
+  case 8:
+    if (is > -1) return 123 / is; // BAD
+  case 9:
+    if (is < 2) return 123 / is; // BAD
+  }
+  if (is != 0) return -1;
+  if (is == 0) type += 1;
+  return 123 / is; // BAD [NOT DETECTED]
+}
+
+int goodTestf3(int type, int met) {
+  int is = getSize(type);
+  if (is == 0) return -1;
+  switch (met) {
+  case 1:
+    if (is < 0) return 123 / is; // GOOD
+  case 2:
+    if (!is && 123 / is) // GOOD 
+      return 123;
+  case 3:
+    if (!is || 123 / is) // GOOD
+      return 123;
+  case 8:
+    if (is < -1) return 123 / is; // GOOD
+  case 9:
+    if (is > 2) return 123 / is; // GOOD
+  }
+  return 123 / is;
+}
+
+int goodTestf3a(int type, int met) {
+  int is = getSize(type);
+  switch (met) {
+  case 1:
+    if (is < 0)
+      return 123 / is; // GOOD
+  case 2:
+    if (!is && 123 / is) // GOOD 
+      return 123;
+  case 3:
+    if (!is || 123 / is) // GOOD 
+      return 123;
+  }
+  return 1;
+}
+
+int badTestf4(int type) {
+  int is = getSize(type);
+  int d;
+  d = type * is;
+  return 123 / d; // BAD
+}
+
+int badTestf5(int type) {
+  int is = getSize(type);
+  int d;
+  d = is / type;
+  return 123 / d; // BAD
+}
+int badTestf6(int type) {
+  int is = getSize(type);
+  int d;
+  d = is / type;
+  return type * 123 / d; // BAD
+}
+
+int badTestf7(int type, int met) {
+  int is = getSize(type);
+  if (is == 0) goto quit;
+  switch (met) {
+  case 1:
+    if (is < 0)
+      return 123 / is; // GOOD
+  }
+  quit:
+    return 123 / is; // BAD
+}
+
+int goodTestf7(int type, int met) {
+  int is = getSize(type);
+  if (is == 0) goto quit2;
+  if (is == 0.) return -1;
+  switch (met) {
+  case 1:
+    if (is < 0.)
+      return 123 / is; // GOOD
+  }
+  return 123 / is; // GOOD
+  quit2:
+    return -1;
+}
+
+int badTestf8(int type) {
+  int is = getSize(type);
+  type /= is; // BAD
+  type %= is; // BAD
+  return type;
+}
+
+float getSizeFloat(float type) {
+  float st;
+  if (type)
+    st = 1.0;
+  else
+    st = 0.0;
+  return st;
+}
+float badTestf9(float type) {
+  float is = getSizeFloat(type);
+  return 123 / is; // BAD
+}
+float goodTestf9(float type) {
+  float is = getSizeFloat(type);
+  if (is == 0.0) return -1;
+  return 123 / is; // GOOD
+}
+
+int badTestf10(int type) {
+  int out = type;
+  int is = getSize(type);
+  if (is > -2) {
+    out /= 123 / (is + 1); // BAD
+  }
+  if (is > 0) {
+    return 123 / (is - 1); // BAD
+  }
+  if (is <= 0) return 0;
+  return 123 / (is - 1); // BAD
+  return 0;
+}
+int badTestf11(int type) {
+  int is = getSize(type);
+  return 123 / (is - 3); // BAD
+}
+
+int goodTestf11(int type) {
+  int is = getSize(type);
+  if (is > 1) {
+    return 123 / (is - 1); // GOOD
+  } else {
+    return 0;
+  }
+}
+
+int badTestf12(FILE * f) {
+  int a;
+  int ret = -1;
+  a = getc(f);
+  if (a == 0) ret = 123 / a; // BAD [NOT DETECTED]
+  return ret;
+}
+
+int goodTestf12(FILE * f) {
+  int a;
+  int ret = -1;
+  a = getc(f);
+  if (a != 0) ret = 123 / a; // GOOD
+  return ret;
+}
+
+int badMyDiv(int type, int is) {
+  type /= is;
+  type %= is;
+  return type;
+}
+
+int goodMyDiv(int type, int is) {
+  if (is == 0) return -1;
+  type /= is;
+  type %= is;
+  return type;
+}
+int badMySubDiv(int type, int is) {
+  type /= (is - 3);
+  type %= (is + 1);
+  return type;
+}
+
+void badTestf13(int type) {
+  int is = getSize(type);
+  badMyDiv(type, is); // BAD
+  badMyDiv(type, is - 2); // BAD
+  badMySubDiv(type, is); // BAD
+  goodMyDiv(type, is); // GOOD
+  if (is < 5)
+    badMySubDiv(type, is); // BAD
+  if (is < 0)
+    badMySubDiv(type, is); // BAD [NOT DETECTED]
+  if (is > 5)
+    badMySubDiv(type, is); // GOOD
+  if (is == 0)
+    badMyDiv(type, is); // BAD
+  if (is > 0)
+    badMyDiv(type, is); // GOOD
+  if (is < 5)
+    badMyDiv(type, is - 3); // BAD
+  if (is < 0)
+    badMyDiv(type, is + 1); // BAD
+  if (is > 5)
+    badMyDiv(type, is - 3); // GOOD
+}

cpp/ql/test/library-tests/CPP-205/elements.expected

@@ -1,14 +1,14 @@
 | CPP-205.cpp:0:0:0:0 | CPP-205.cpp |  |
 | CPP-205.cpp:1:20:1:20 | T |  |
 | CPP-205.cpp:1:20:1:20 | definition of T |  |
-| CPP-205.cpp:2:5:2:5 | definition of fn | function declaration entry for fn<int>(int) -> int |
-| CPP-205.cpp:2:5:2:5 | fn | function fn<int>(int) -> int |
-| CPP-205.cpp:2:5:2:6 | definition of fn | function declaration entry for fn<T>(T) -> int |
-| CPP-205.cpp:2:5:2:6 | fn | function fn<T>(T) -> int |
-| CPP-205.cpp:2:10:2:12 | definition of out | parameter declaration entry for fn<T>(T) -> int |
-| CPP-205.cpp:2:10:2:12 | definition of out | parameter declaration entry for fn<int>(int) -> int |
-| CPP-205.cpp:2:10:2:12 | out | parameter for fn<T>(T) -> int |
-| CPP-205.cpp:2:10:2:12 | out | parameter for fn<int>(int) -> int |
+| CPP-205.cpp:2:5:2:5 | definition of fn | function declaration entry for int fn<int>(int) |
+| CPP-205.cpp:2:5:2:5 | fn | function int fn<int>(int) |
+| CPP-205.cpp:2:5:2:6 | definition of fn | function declaration entry for int fn<T>(T) |
+| CPP-205.cpp:2:5:2:6 | fn | function int fn<T>(T) |
+| CPP-205.cpp:2:10:2:12 | definition of out | parameter declaration entry for int fn<T>(T) |
+| CPP-205.cpp:2:10:2:12 | definition of out | parameter declaration entry for int fn<int>(int) |
+| CPP-205.cpp:2:10:2:12 | out | parameter for int fn<T>(T) |
+| CPP-205.cpp:2:10:2:12 | out | parameter for int fn<int>(int) |
 | CPP-205.cpp:2:15:5:1 | { ... } |  |
 | CPP-205.cpp:2:15:5:1 | { ... } |  |
 | CPP-205.cpp:3:3:3:33 | declaration |  |
@@ -20,16 +20,16 @@
 | CPP-205.cpp:4:3:4:11 | return ... |  |
 | CPP-205.cpp:4:10:4:10 | 0 |  |
 | CPP-205.cpp:4:10:4:10 | 0 |  |
-| CPP-205.cpp:7:5:7:8 | definition of main | function declaration entry for main() -> int |
-| CPP-205.cpp:7:5:7:8 | main | function main() -> int |
+| CPP-205.cpp:7:5:7:8 | definition of main | function declaration entry for int main() |
+| CPP-205.cpp:7:5:7:8 | main | function int main() |
 | CPP-205.cpp:7:12:9:1 | { ... } |  |
 | CPP-205.cpp:8:3:8:15 | return ... |  |
 | CPP-205.cpp:8:10:8:11 | call to fn |  |
 | CPP-205.cpp:8:13:8:13 | 0 |  |
-| file://:0:0:0:0 | (unnamed parameter 0) | parameter for __va_list_tag::operator=(__va_list_tag &&) -> __va_list_tag & |
-| file://:0:0:0:0 | (unnamed parameter 0) | parameter for __va_list_tag::operator=(const __va_list_tag &) -> __va_list_tag & |
+| file://:0:0:0:0 | (unnamed parameter 0) | parameter for __va_list_tag& __va_list_tag::operator=(__va_list_tag const&) |
+| file://:0:0:0:0 | (unnamed parameter 0) | parameter for __va_list_tag& __va_list_tag::operator=(__va_list_tag&&) |
 | file://:0:0:0:0 | __super |  |
 | file://:0:0:0:0 | __va_list_tag |  |
-| file://:0:0:0:0 | operator= | function __va_list_tag::operator=(__va_list_tag &&) -> __va_list_tag & |
-| file://:0:0:0:0 | operator= | function __va_list_tag::operator=(const __va_list_tag &) -> __va_list_tag & |
+| file://:0:0:0:0 | operator= | function __va_list_tag& __va_list_tag::operator=(__va_list_tag const&) |
+| file://:0:0:0:0 | operator= | function __va_list_tag& __va_list_tag::operator=(__va_list_tag&&) |
 | file://:0:0:0:0 | y |  |

cpp/ql/test/library-tests/CPP-205/elements.ql

@@ -1,17 +1,19 @@
 import cpp
+import semmle.code.cpp.Print
 
 string describe(Element e) {
-  result = "function " + e.(Function).getFullSignature()
+  e instanceof Function and
+  result = "function " + getIdentityString(e)
   or
   result =
     "function declaration entry for " +
-      e.(FunctionDeclarationEntry).getFunction().getFullSignature()
+      getIdentityString(e.(FunctionDeclarationEntry).getFunction())
   or
-  result = "parameter for " + e.(Parameter).getFunction().getFullSignature()
+  result = "parameter for " + getIdentityString(e.(Parameter).getFunction())
   or
   result =
     "parameter declaration entry for " +
-      e.(ParameterDeclarationEntry).getFunctionDeclarationEntry().getFunction().getFullSignature()
+      getIdentityString(e.(ParameterDeclarationEntry).getFunctionDeclarationEntry().getFunction())
 }
 
 from Element e

cpp/ql/test/library-tests/allocators/allocators.expected

@@ -1,51 +1,51 @@
 newExprs
-| allocators.cpp:49:3:49:9 | new | int | operator new(unsigned long) -> void * | 4 | 4 |  |  |
-| allocators.cpp:50:3:50:15 | new | int | operator new(size_t, float) -> void * | 4 | 4 |  |  |
-| allocators.cpp:51:3:51:11 | new | int | operator new(unsigned long) -> void * | 4 | 4 |  |  |
-| allocators.cpp:52:3:52:14 | new | String | operator new(unsigned long) -> void * | 8 | 8 |  |  |
-| allocators.cpp:53:3:53:27 | new | String | operator new(size_t, float) -> void * | 8 | 8 |  |  |
-| allocators.cpp:54:3:54:17 | new | Overaligned | operator new(unsigned long, align_val_t) -> void * | 256 | 128 | aligned |  |
-| allocators.cpp:55:3:55:25 | new | Overaligned | operator new(size_t, align_val_t, float) -> void * | 256 | 128 | aligned |  |
-| allocators.cpp:107:3:107:18 | new | FailedInit | FailedInit::operator new(size_t) -> void * | 1 | 1 |  |  |
-| allocators.cpp:109:3:109:35 | new | FailedInitOveraligned | FailedInitOveraligned::operator new(size_t, align_val_t, float) -> void * | 128 | 128 | aligned |  |
-| allocators.cpp:129:3:129:21 | new | int | operator new(size_t, void *) -> void * | 4 | 4 |  | & ... |
-| allocators.cpp:135:3:135:26 | new | int | operator new(size_t, const nothrow_t &) -> void * | 4 | 4 |  |  |
+| allocators.cpp:49:3:49:9 | new | int | void* operator new(unsigned long) | 4 | 4 |  |  |
+| allocators.cpp:50:3:50:15 | new | int | void* operator new(size_t, float) | 4 | 4 |  |  |
+| allocators.cpp:51:3:51:11 | new | int | void* operator new(unsigned long) | 4 | 4 |  |  |
+| allocators.cpp:52:3:52:14 | new | String | void* operator new(unsigned long) | 8 | 8 |  |  |
+| allocators.cpp:53:3:53:27 | new | String | void* operator new(size_t, float) | 8 | 8 |  |  |
+| allocators.cpp:54:3:54:17 | new | Overaligned | void* operator new(unsigned long, std::align_val_t) | 256 | 128 | aligned |  |
+| allocators.cpp:55:3:55:25 | new | Overaligned | void* operator new(size_t, std::align_val_t, float) | 256 | 128 | aligned |  |
+| allocators.cpp:107:3:107:18 | new | FailedInit | void* FailedInit::operator new(size_t) | 1 | 1 |  |  |
+| allocators.cpp:109:3:109:35 | new | FailedInitOveraligned | void* FailedInitOveraligned::operator new(size_t, std::align_val_t, float) | 128 | 128 | aligned |  |
+| allocators.cpp:129:3:129:21 | new | int | void* operator new(std::size_t, void*) | 4 | 4 |  | & ... |
+| allocators.cpp:135:3:135:26 | new | int | void* operator new(std::size_t, std::nothrow_t const&) | 4 | 4 |  |  |
 newArrayExprs
-| allocators.cpp:68:3:68:12 | new[] | int[] | int | operator new[](unsigned long) -> void * | 4 | 4 |  | n |  |
-| allocators.cpp:69:3:69:18 | new[] | int[] | int | operator new[](size_t, float) -> void * | 4 | 4 |  | n |  |
-| allocators.cpp:70:3:70:15 | new[] | String[] | String | operator new[](unsigned long) -> void * | 8 | 8 |  | n |  |
-| allocators.cpp:71:3:71:20 | new[] | Overaligned[] | Overaligned | operator new[](unsigned long, align_val_t) -> void * | 256 | 128 | aligned | n |  |
-| allocators.cpp:72:3:72:16 | new[] | String[10] | String | operator new[](unsigned long) -> void * | 8 | 8 |  |  |  |
-| allocators.cpp:108:3:108:19 | new[] | FailedInit[] | FailedInit | FailedInit::operator new[](size_t) -> void * | 1 | 1 |  | n |  |
-| allocators.cpp:110:3:110:37 | new[] | FailedInitOveraligned[10] | FailedInitOveraligned | FailedInitOveraligned::operator new[](size_t, align_val_t, float) -> void * | 128 | 128 | aligned |  |  |
-| allocators.cpp:132:3:132:17 | new[] | int[1] | int | operator new[](size_t, void *) -> void * | 4 | 4 |  |  | buf |
-| allocators.cpp:136:3:136:26 | new[] | int[2] | int | operator new[](size_t, const nothrow_t &) -> void * | 4 | 4 |  |  |  |
-| allocators.cpp:142:13:142:27 | new[] | char[][10] | char[10] | operator new[](unsigned long) -> void * | 10 | 1 |  | x |  |
-| allocators.cpp:143:13:143:28 | new[] | char[20][20] | char[20] | operator new[](unsigned long) -> void * | 20 | 1 |  |  |  |
-| allocators.cpp:144:13:144:31 | new[] | char[][30][30] | char[30][30] | operator new[](unsigned long) -> void * | 900 | 1 |  | x |  |
+| allocators.cpp:68:3:68:12 | new[] | int[] | int | void* operator new[](unsigned long) | 4 | 4 |  | n |  |
+| allocators.cpp:69:3:69:18 | new[] | int[] | int | void* operator new[](size_t, float) | 4 | 4 |  | n |  |
+| allocators.cpp:70:3:70:15 | new[] | String[] | String | void* operator new[](unsigned long) | 8 | 8 |  | n |  |
+| allocators.cpp:71:3:71:20 | new[] | Overaligned[] | Overaligned | void* operator new[](unsigned long, std::align_val_t) | 256 | 128 | aligned | n |  |
+| allocators.cpp:72:3:72:16 | new[] | String[10] | String | void* operator new[](unsigned long) | 8 | 8 |  |  |  |
+| allocators.cpp:108:3:108:19 | new[] | FailedInit[] | FailedInit | void* FailedInit::operator new[](size_t) | 1 | 1 |  | n |  |
+| allocators.cpp:110:3:110:37 | new[] | FailedInitOveraligned[10] | FailedInitOveraligned | void* FailedInitOveraligned::operator new[](size_t, std::align_val_t, float) | 128 | 128 | aligned |  |  |
+| allocators.cpp:132:3:132:17 | new[] | int[1] | int | void* operator new[](std::size_t, void*) | 4 | 4 |  |  | buf |
+| allocators.cpp:136:3:136:26 | new[] | int[2] | int | void* operator new[](std::size_t, std::nothrow_t const&) | 4 | 4 |  |  |  |
+| allocators.cpp:142:13:142:27 | new[] | char[][10] | char[10] | void* operator new[](unsigned long) | 10 | 1 |  | x |  |
+| allocators.cpp:143:13:143:28 | new[] | char[20][20] | char[20] | void* operator new[](unsigned long) | 20 | 1 |  |  |  |
+| allocators.cpp:144:13:144:31 | new[] | char[][30][30] | char[30][30] | void* operator new[](unsigned long) | 900 | 1 |  | x |  |
 newExprDeallocators
-| allocators.cpp:52:3:52:14 | new | String | operator delete(void *, unsigned long) -> void | 8 | 8 | sized  |
-| allocators.cpp:53:3:53:27 | new | String | operator delete(void *, float) -> void | 8 | 8 |   |
-| allocators.cpp:107:3:107:18 | new | FailedInit | FailedInit::operator delete(void *, size_t) -> void | 1 | 1 | sized  |
-| allocators.cpp:109:3:109:35 | new | FailedInitOveraligned | FailedInitOveraligned::operator delete(void *, align_val_t, float) -> void | 128 | 128 |  aligned |
+| allocators.cpp:52:3:52:14 | new | String | void operator delete(void*, unsigned long) | 8 | 8 | sized  |
+| allocators.cpp:53:3:53:27 | new | String | void operator delete(void*, float) | 8 | 8 |   |
+| allocators.cpp:107:3:107:18 | new | FailedInit | void FailedInit::operator delete(void*, size_t) | 1 | 1 | sized  |
+| allocators.cpp:109:3:109:35 | new | FailedInitOveraligned | void FailedInitOveraligned::operator delete(void*, std::align_val_t, float) | 128 | 128 |  aligned |
 newArrayExprDeallocators
-| allocators.cpp:70:3:70:15 | new[] | String | operator delete[](void *, unsigned long) -> void | 8 | 8 | sized  |
-| allocators.cpp:72:3:72:16 | new[] | String | operator delete[](void *, unsigned long) -> void | 8 | 8 | sized  |
-| allocators.cpp:108:3:108:19 | new[] | FailedInit | FailedInit::operator delete[](void *, size_t) -> void | 1 | 1 | sized  |
-| allocators.cpp:110:3:110:37 | new[] | FailedInitOveraligned | FailedInitOveraligned::operator delete[](void *, align_val_t, float) -> void | 128 | 128 |  aligned |
+| allocators.cpp:70:3:70:15 | new[] | String | void operator delete[](void*, unsigned long) | 8 | 8 | sized  |
+| allocators.cpp:72:3:72:16 | new[] | String | void operator delete[](void*, unsigned long) | 8 | 8 | sized  |
+| allocators.cpp:108:3:108:19 | new[] | FailedInit | void FailedInit::operator delete[](void*, size_t) | 1 | 1 | sized  |
+| allocators.cpp:110:3:110:37 | new[] | FailedInitOveraligned | void FailedInitOveraligned::operator delete[](void*, std::align_val_t, float) | 128 | 128 |  aligned |
 deleteExprs
-| allocators.cpp:59:3:59:35 | delete | int | operator delete(void *, unsigned long) -> void | 4 | 4 | sized  |
-| allocators.cpp:60:3:60:38 | delete | String | operator delete(void *, unsigned long) -> void | 8 | 8 | sized  |
-| allocators.cpp:61:3:61:44 | delete | SizedDealloc | SizedDealloc::operator delete(void *, size_t) -> void | 32 | 1 | sized  |
-| allocators.cpp:62:3:62:43 | delete | Overaligned | operator delete(void *, unsigned long, align_val_t) -> void | 256 | 128 | sized aligned |
-| allocators.cpp:64:3:64:44 | delete | const String | operator delete(void *, unsigned long) -> void | 8 | 8 | sized  |
+| allocators.cpp:59:3:59:35 | delete | int | void operator delete(void*, unsigned long) | 4 | 4 | sized  |
+| allocators.cpp:60:3:60:38 | delete | String | void operator delete(void*, unsigned long) | 8 | 8 | sized  |
+| allocators.cpp:61:3:61:44 | delete | SizedDealloc | void SizedDealloc::operator delete(void*, size_t) | 32 | 1 | sized  |
+| allocators.cpp:62:3:62:43 | delete | Overaligned | void operator delete(void*, unsigned long, std::align_val_t) | 256 | 128 | sized aligned |
+| allocators.cpp:64:3:64:44 | delete | const String | void operator delete(void*, unsigned long) | 8 | 8 | sized  |
 deleteArrayExprs
-| allocators.cpp:78:3:78:37 | delete[] | int | operator delete[](void *, unsigned long) -> void | 4 | 4 | sized  |
-| allocators.cpp:79:3:79:40 | delete[] | String | operator delete[](void *, unsigned long) -> void | 8 | 8 | sized  |
-| allocators.cpp:80:3:80:46 | delete[] | SizedDealloc | SizedDealloc::operator delete[](void *, size_t) -> void | 32 | 1 | sized  |
-| allocators.cpp:81:3:81:45 | delete[] | Overaligned | operator delete[](void *, unsigned long, align_val_t) -> void | 256 | 128 | sized aligned |
-| allocators.cpp:82:3:82:49 | delete[] | PolymorphicBase | operator delete[](void *, unsigned long) -> void | 8 | 8 | sized  |
-| allocators.cpp:83:3:83:23 | delete[] | int | operator delete[](void *, unsigned long) -> void | 4 | 4 | sized  |
+| allocators.cpp:78:3:78:37 | delete[] | int | void operator delete[](void*, unsigned long) | 4 | 4 | sized  |
+| allocators.cpp:79:3:79:40 | delete[] | String | void operator delete[](void*, unsigned long) | 8 | 8 | sized  |
+| allocators.cpp:80:3:80:46 | delete[] | SizedDealloc | void SizedDealloc::operator delete[](void*, size_t) | 32 | 1 | sized  |
+| allocators.cpp:81:3:81:45 | delete[] | Overaligned | void operator delete[](void*, unsigned long, std::align_val_t) | 256 | 128 | sized aligned |
+| allocators.cpp:82:3:82:49 | delete[] | PolymorphicBase | void operator delete[](void*, unsigned long) | 8 | 8 | sized  |
+| allocators.cpp:83:3:83:23 | delete[] | int | void operator delete[](void*, unsigned long) | 4 | 4 | sized  |
 allocationFunctions
 | allocators.cpp:7:7:7:18 | operator new | getSizeArg = 0, requiresDealloc |
 | allocators.cpp:8:7:8:20 | operator new[] | getSizeArg = 0, requiresDealloc |

cpp/ql/test/library-tests/allocators/allocators.ql

@@ -1,12 +1,13 @@
 import cpp
 import semmle.code.cpp.models.implementations.Allocation
+import semmle.code.cpp.Print
 
 query predicate newExprs(
   NewExpr expr, string type, string sig, int size, int alignment, string form, string placement
 ) {
   exists(Function allocator, Type allocatedType |
     expr.getAllocator() = allocator and
-    sig = allocator.getFullSignature() and
+    sig = getIdentityString(allocator) and
     allocatedType = expr.getAllocatedType() and
     type = allocatedType.toString() and
     size = allocatedType.getSize() and
@@ -24,7 +25,7 @@ query predicate newArrayExprs(
 ) {
   exists(Function allocator, Type arrayType, Type elementType |
     expr.getAllocator() = allocator and
-    sig = allocator.getFullSignature() and
+    sig = getIdentityString(allocator) and
     arrayType = expr.getAllocatedType() and
     t1 = arrayType.toString() and
     elementType = expr.getAllocatedElementType() and
@@ -44,7 +45,7 @@ query predicate newExprDeallocators(
 ) {
   exists(Function deallocator, Type allocatedType |
     expr.getDeallocator() = deallocator and
-    sig = deallocator.getFullSignature() and
+    sig = getIdentityString(deallocator) and
     allocatedType = expr.getAllocatedType() and
     type = allocatedType.toString() and
     size = allocatedType.getSize() and
@@ -62,7 +63,7 @@ query predicate newArrayExprDeallocators(
 ) {
   exists(Function deallocator, Type elementType |
     expr.getDeallocator() = deallocator and
-    sig = deallocator.getFullSignature() and
+    sig = getIdentityString(deallocator) and
     elementType = expr.getAllocatedElementType() and
     type = elementType.toString() and
     size = elementType.getSize() and
@@ -80,7 +81,7 @@ query predicate deleteExprs(
 ) {
   exists(Function deallocator, Type deletedType |
     expr.getDeallocator() = deallocator and
-    sig = deallocator.getFullSignature() and
+    sig = getIdentityString(deallocator) and
     deletedType = expr.getDeletedObjectType() and
     type = deletedType.toString() and
     size = deletedType.getSize() and
@@ -98,7 +99,7 @@ query predicate deleteArrayExprs(
 ) {
   exists(Function deallocator, Type elementType |
     expr.getDeallocator() = deallocator and
-    sig = deallocator.getFullSignature() and
+    sig = getIdentityString(deallocator) and
     elementType = expr.getDeletedElementType() and
     type = elementType.toString() and
     size = elementType.getSize() and

cpp/ql/test/library-tests/noexcept/copy_from_prototype/copy_from_prototype.expected

@@ -1,30 +1,30 @@
-| copy_from_prototype.cpp:3:7:3:7 | a | a<int>::a(a<int> &&) -> void | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
-| copy_from_prototype.cpp:3:7:3:7 | a | a<int>::a(const a<int> &) -> void | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
-| copy_from_prototype.cpp:3:7:3:7 | operator= | a<int>::operator=(a<int> &&) -> a<int> & | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
-| copy_from_prototype.cpp:3:7:3:7 | operator= | a<int>::operator=(const a<int> &) -> a<int> & | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
-| copy_from_prototype.cpp:4:26:4:26 | a | a<<unnamed>>::a<(unnamed template parameter)>() -> void | copy_from_prototype.cpp:3:7:3:7 | a<<unnamed>> | 123 |
-| copy_from_prototype.cpp:4:26:4:26 | a | a<int>::a<(unnamed template parameter)>() -> void | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
-| copy_from_prototype.cpp:7:7:7:7 | b | b::b() -> void | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
-| copy_from_prototype.cpp:7:7:7:7 | b | b::b(b &&) -> void | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
-| copy_from_prototype.cpp:7:7:7:7 | b | b::b(const b &) -> void | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
-| copy_from_prototype.cpp:7:7:7:7 | operator= | b::operator=(b &&) -> b & | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
-| copy_from_prototype.cpp:7:7:7:7 | operator= | b::operator=(const b &) -> b & | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
-| copy_from_prototype.cpp:13:7:13:7 | c | c<int>::c(c<int> &&) -> void | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
-| copy_from_prototype.cpp:13:7:13:7 | c | c<int>::c(const c<int> &) -> void | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
-| copy_from_prototype.cpp:13:7:13:7 | operator= | c<int>::operator=(c<int> &&) -> c<int> & | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
-| copy_from_prototype.cpp:13:7:13:7 | operator= | c<int>::operator=(const c<int> &) -> c<int> & | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
-| copy_from_prototype.cpp:14:26:14:26 | c | c<T>::c<(unnamed template parameter)>() -> void | copy_from_prototype.cpp:13:7:13:7 | c<T> | X |
-| copy_from_prototype.cpp:14:26:14:26 | c | c<int>::c<(unnamed template parameter)>() -> void | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
-| copy_from_prototype.cpp:17:7:17:7 | d | d::d() -> void | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
-| copy_from_prototype.cpp:17:7:17:7 | d | d::d(const d &) -> void | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
-| copy_from_prototype.cpp:17:7:17:7 | d | d::d(d &&) -> void | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
-| copy_from_prototype.cpp:17:7:17:7 | operator= | d::operator=(const d &) -> d & | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
-| copy_from_prototype.cpp:17:7:17:7 | operator= | d::operator=(d &&) -> d & | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
-| copy_from_prototype.cpp:22:8:22:8 | e | e<int>::e(const e<int> &) -> void | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
-| copy_from_prototype.cpp:22:8:22:8 | e | e<int>::e(e<int> &&) -> void | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
-| copy_from_prototype.cpp:22:8:22:8 | operator= | e<int>::operator=(const e<int> &) -> e<int> & | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
-| copy_from_prototype.cpp:22:8:22:8 | operator= | e<int>::operator=(e<int> &&) -> e<int> & | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
-| copy_from_prototype.cpp:23:26:23:26 | e | e<T>::e<(unnamed template parameter)>() -> void | copy_from_prototype.cpp:22:8:22:8 | e<T> | 456 |
-| copy_from_prototype.cpp:26:35:26:43 | e | e<int>::e<(unnamed template parameter)>() -> void | copy_from_prototype.cpp:22:8:22:8 | e<int> | 456 |
-| file://:0:0:0:0 | operator= | __va_list_tag::operator=(__va_list_tag &&) -> __va_list_tag & | file://:0:0:0:0 | __va_list_tag | <none> |
-| file://:0:0:0:0 | operator= | __va_list_tag::operator=(const __va_list_tag &) -> __va_list_tag & | file://:0:0:0:0 | __va_list_tag | <none> |
+| copy_from_prototype.cpp:3:7:3:7 | a | void a<int>::a(a<int> const&) | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
+| copy_from_prototype.cpp:3:7:3:7 | a | void a<int>::a(a<int>&&) | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
+| copy_from_prototype.cpp:3:7:3:7 | operator= | a<int>& a<int>::operator=(a<int> const&) | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
+| copy_from_prototype.cpp:3:7:3:7 | operator= | a<int>& a<int>::operator=(a<int>&&) | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
+| copy_from_prototype.cpp:4:26:4:26 | a | void a<(unnamed template parameter)>::a<(unnamed template parameter)>() | copy_from_prototype.cpp:3:7:3:7 | a<<unnamed>> | 123 |
+| copy_from_prototype.cpp:4:26:4:26 | a | void a<int>::a<(unnamed template parameter)>() | copy_from_prototype.cpp:3:7:3:7 | a<int> | <no expr> |
+| copy_from_prototype.cpp:7:7:7:7 | b | void b::b() | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
+| copy_from_prototype.cpp:7:7:7:7 | b | void b::b(b const&) | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
+| copy_from_prototype.cpp:7:7:7:7 | b | void b::b(b&&) | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
+| copy_from_prototype.cpp:7:7:7:7 | operator= | b& b::operator=(b const&) | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
+| copy_from_prototype.cpp:7:7:7:7 | operator= | b& b::operator=(b&&) | copy_from_prototype.cpp:7:7:7:7 | b | <no expr> |
+| copy_from_prototype.cpp:13:7:13:7 | c | void c<int>::c(c<int> const&) | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
+| copy_from_prototype.cpp:13:7:13:7 | c | void c<int>::c(c<int>&&) | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
+| copy_from_prototype.cpp:13:7:13:7 | operator= | c<int>& c<int>::operator=(c<int> const&) | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
+| copy_from_prototype.cpp:13:7:13:7 | operator= | c<int>& c<int>::operator=(c<int>&&) | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
+| copy_from_prototype.cpp:14:26:14:26 | c | void c<T>::c<(unnamed template parameter)>() | copy_from_prototype.cpp:13:7:13:7 | c<T> | X |
+| copy_from_prototype.cpp:14:26:14:26 | c | void c<int>::c<(unnamed template parameter)>() | copy_from_prototype.cpp:13:7:13:7 | c<int> | <no expr> |
+| copy_from_prototype.cpp:17:7:17:7 | d | void d::d() | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
+| copy_from_prototype.cpp:17:7:17:7 | d | void d::d(d const&) | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
+| copy_from_prototype.cpp:17:7:17:7 | d | void d::d(d&&) | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
+| copy_from_prototype.cpp:17:7:17:7 | operator= | d& d::operator=(d const&) | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
+| copy_from_prototype.cpp:17:7:17:7 | operator= | d& d::operator=(d&&) | copy_from_prototype.cpp:17:7:17:7 | d | <no expr> |
+| copy_from_prototype.cpp:22:8:22:8 | e | void e<int>::e(e<int> const&) | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
+| copy_from_prototype.cpp:22:8:22:8 | e | void e<int>::e(e<int>&&) | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
+| copy_from_prototype.cpp:22:8:22:8 | operator= | e<int>& e<int>::operator=(e<int> const&) | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
+| copy_from_prototype.cpp:22:8:22:8 | operator= | e<int>& e<int>::operator=(e<int>&&) | copy_from_prototype.cpp:22:8:22:8 | e<int> | <no expr> |
+| copy_from_prototype.cpp:23:26:23:26 | e | void e<T>::e<(unnamed template parameter)>() | copy_from_prototype.cpp:22:8:22:8 | e<T> | 456 |
+| copy_from_prototype.cpp:26:35:26:43 | e | void e<int>::e<(unnamed template parameter)>() | copy_from_prototype.cpp:22:8:22:8 | e<int> | 456 |
+| file://:0:0:0:0 | operator= | __va_list_tag& __va_list_tag::operator=(__va_list_tag const&) | file://:0:0:0:0 | __va_list_tag | <none> |
+| file://:0:0:0:0 | operator= | __va_list_tag& __va_list_tag::operator=(__va_list_tag&&) | file://:0:0:0:0 | __va_list_tag | <none> |

cpp/ql/test/library-tests/noexcept/copy_from_prototype/copy_from_prototype.ql

@@ -1,4 +1,5 @@
 import cpp
+import semmle.code.cpp.Print
 
 from Function f, string e
 where
@@ -8,4 +9,4 @@ where
     then e = f.getADeclarationEntry().getNoExceptExpr().toString()
     else e = "<no expr>"
   else e = "<none>"
-select f, f.getFullSignature(), f.getDeclaringType(), e
+select f, getIdentityString(f), f.getDeclaringType(), e

cpp/ql/test/query-tests/Critical/MissingCheckScanf/MissingCheckScanf.expected

@@ -19,3 +19,6 @@
 | test.cpp:302:8:302:12 | ptr_i | This variable is read, but may not have been written. It should be guarded by a check that the $@ returns at least 1. | test.cpp:301:3:301:7 | call to scanf | call to scanf |
 | test.cpp:310:7:310:7 | i | This variable is read, but may not have been written. It should be guarded by a check that the $@ returns at least 1. | test.cpp:309:3:309:7 | call to scanf | call to scanf |
 | test.cpp:404:25:404:25 | u | This variable is read, but may not have been written. It should be guarded by a check that the $@ returns at least 1. | test.cpp:403:6:403:11 | call to sscanf | call to sscanf |
+| test.cpp:416:7:416:7 | i | This variable is read, but may not have been written. It should be guarded by a check that the $@ returns at least 1. | test.cpp:413:7:413:11 | call to scanf | call to scanf |
+| test.cpp:423:7:423:7 | i | This variable is read, but may not have been written. It should be guarded by a check that the $@ returns at least 1. | test.cpp:420:7:420:11 | call to scanf | call to scanf |
+| test.cpp:430:6:430:6 | i | This variable is read, but may not have been written. It should be guarded by a check that the $@ returns at least 1. | test.cpp:429:2:429:6 | call to scanf | call to scanf |

cpp/ql/test/query-tests/Critical/MissingCheckScanf/test.cpp

@@ -406,3 +406,26 @@ char *my_string_copy() {
 	*ptr++ = 0;
 	return DST_STRING;
 }
+
+void scan_and_write() {
+	{
+		int i;
+		if (scanf("%d", &i) < 1) {
+			i = 0;
+		}
+		use(i);  // GOOD [FALSE POSITIVE]: variable is overwritten with a default value when scanf fails
+	}
+	{
+		int i;
+		if (scanf("%d", &i) != 1) {
+			i = 0;
+		}
+		use(i);  // GOOD [FALSE POSITIVE]: variable is overwritten with a default value when scanf fails
+	}
+}
+
+void scan_and_static_variable() {
+	static int i;
+	scanf("%d", &i);
+	use(i);  // GOOD [FALSE POSITIVE]: static variables are always 0-initialized
+}

cpp/ql/test/query-tests/Security/CWE/CWE-078/semmle/ExecTainted/ExecTainted.expected

@@ -71,8 +71,6 @@ edges
 | test.cpp:220:10:220:16 | strncat output argument | test.cpp:222:32:222:38 | command indirection |
 | test.cpp:220:19:220:26 | filename indirection | test.cpp:220:10:220:16 | strncat output argument |
 | test.cpp:220:19:220:26 | filename indirection | test.cpp:220:10:220:16 | strncat output argument |
-| test.cpp:220:19:220:26 | filename indirection | test.cpp:220:10:220:16 | strncat output argument |
-| test.cpp:220:19:220:26 | filename indirection | test.cpp:220:10:220:16 | strncat output argument |
 nodes
 | test.cpp:15:27:15:30 | argv | semmle.label | argv |
 | test.cpp:22:13:22:20 | sprintf output argument | semmle.label | sprintf output argument |
@@ -151,6 +149,7 @@ nodes
 | test.cpp:220:19:220:26 | filename indirection | semmle.label | filename indirection |
 | test.cpp:220:19:220:26 | filename indirection | semmle.label | filename indirection |
 | test.cpp:222:32:222:38 | command indirection | semmle.label | command indirection |
+| test.cpp:222:32:222:38 | command indirection | semmle.label | command indirection |
 subpaths
 | test.cpp:196:26:196:33 | filename | test.cpp:186:47:186:54 | filename | test.cpp:188:11:188:17 | command [post update] | test.cpp:196:10:196:16 | command [post update] |
 | test.cpp:196:26:196:33 | filename | test.cpp:186:47:186:54 | filename | test.cpp:188:11:188:17 | command [post update] | test.cpp:196:10:196:16 | command [post update] |

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -1,5 +1,6 @@
 using Xunit;
 using Semmle.Autobuild.Shared;
+using Semmle.Util;
 using System.Collections.Generic;
 using System;
 using System.Linq;
@@ -85,6 +86,15 @@ namespace Semmle.Autobuild.CSharp.Tests
             return ret;
         }
 
+        int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
+        {
+            var ret = (this as IBuildActions).RunProcess(cmd, args, workingDirectory, env, out var stdout);
+
+            stdout.ForEach(line => onOutput(line));
+
+            return ret;
+        }
+
         public IList<string> DirectoryDeleteIn { get; } = new List<string>();
 
         void IBuildActions.DirectoryDelete(string dir, bool recursive)
@@ -145,6 +155,14 @@ namespace Semmle.Autobuild.CSharp.Tests
 
         bool IBuildActions.IsWindows() => IsWindows;
 
+        public bool IsMacOs { get; set; }
+
+        bool IBuildActions.IsMacOs() => IsMacOs;
+
+        public bool IsArm { get; set; }
+
+        bool IBuildActions.IsArm() => IsArm;
+
         public string PathCombine(params string[] parts)
         {
             return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));
@@ -192,6 +210,16 @@ namespace Semmle.Autobuild.CSharp.Tests
             if (!DownloadFiles.Contains((address, fileName)))
                 throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");
         }
+
+
+        public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new TestDiagnosticWriter();
+    }
+
+    internal class TestDiagnosticWriter : IDiagnosticsWriter
+    {
+        public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
+
+        public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
     }
 
     /// <summary>
@@ -383,6 +411,7 @@ namespace Semmle.Autobuild.CSharp.Tests
             actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
             actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
+            actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = isWindows ? "win64" : "linux64";
             actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;

csharp/autobuilder/Semmle.Autobuild.CSharp/AutoBuildRule.cs

@@ -0,0 +1,69 @@
+using Semmle.Autobuild.Shared;
+using Semmle.Extraction.CSharp;
+
+namespace Semmle.Autobuild.CSharp
+{
+    internal class AutoBuildRule : IBuildRule<CSharpAutobuildOptions>
+    {
+        private readonly CSharpAutobuilder autobuilder;
+
+        public DotNetRule DotNetRule { get; }
+
+        public MsBuildRule MsBuildRule { get; }
+
+        public BuildCommandAutoRule BuildCommandAutoRule { get; }
+
+        public AutoBuildRule(CSharpAutobuilder autobuilder)
+        {
+            this.autobuilder = autobuilder;
+            this.DotNetRule = new DotNetRule();
+            this.MsBuildRule = new MsBuildRule();
+            this.BuildCommandAutoRule = new BuildCommandAutoRule(DotNetRule.WithDotNet);
+        }
+
+        public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
+        {
+            var cleanTrapFolder =
+                BuildScript.DeleteDirectory(this.autobuilder.TrapDir);
+            var cleanSourceArchive =
+                BuildScript.DeleteDirectory(this.autobuilder.SourceArchiveDir);
+            var tryCleanExtractorArgsLogs =
+                BuildScript.Create(actions =>
+                {
+                    foreach (var file in Extractor.GetCSharpArgsLogs())
+                    {
+                        try
+                        {
+                            actions.FileDelete(file);
+                        }
+                        catch // lgtm[cs/catch-of-all-exceptions] lgtm[cs/empty-catch-block]
+                        { }
+                    }
+
+                    return 0;
+                });
+            var attemptExtractorCleanup =
+                BuildScript.Try(cleanTrapFolder) &
+                BuildScript.Try(cleanSourceArchive) &
+                tryCleanExtractorArgsLogs &
+                BuildScript.DeleteFile(Extractor.GetCSharpLogPath());
+
+            /// <summary>
+            /// Execute script `s` and check that the C# extractor has been executed.
+            /// If either fails, attempt to cleanup any artifacts produced by the extractor,
+            /// and exit with code 1, in order to proceed to the next attempt.
+            /// </summary>
+            BuildScript IntermediateAttempt(BuildScript s) =>
+                (s & this.autobuilder.CheckExtractorRun(false)) |
+                (attemptExtractorCleanup & BuildScript.Failure);
+
+            return
+                // First try .NET Core
+                IntermediateAttempt(this.DotNetRule.Analyse(this.autobuilder, true)) |
+                // Then MSBuild
+                (() => IntermediateAttempt(this.MsBuildRule.Analyse(this.autobuilder, true))) |
+                // And finally look for a script that might be a build script
+                (() => this.BuildCommandAutoRule.Analyse(this.autobuilder, true) & this.autobuilder.CheckExtractorRun(true));
+        }
+    }
+}

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

@@ -1,6 +1,8 @@
 using Semmle.Extraction.CSharp;
 using Semmle.Util.Logging;
 using Semmle.Autobuild.Shared;
+using Semmle.Util;
+using System.Linq;
 
 namespace Semmle.Autobuild.CSharp
 {
@@ -29,25 +31,16 @@ namespace Semmle.Autobuild.CSharp
 
     public class CSharpAutobuilder : Autobuilder<CSharpAutobuildOptions>
     {
-        public CSharpAutobuilder(IBuildActions actions, CSharpAutobuildOptions options) : base(actions, options) { }
+        private const string buildCommandDocsUrl =
+            "https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages";
+
+        private readonly AutoBuildRule autoBuildRule;
+
+        public CSharpAutobuilder(IBuildActions actions, CSharpAutobuildOptions options) : base(actions, options, new CSharpDiagnosticClassifier()) =>
+            this.autoBuildRule = new AutoBuildRule(this);
 
         public override BuildScript GetBuildScript()
         {
-            /// <summary>
-            /// A script that checks that the C# extractor has been executed.
-            /// </summary>
-            BuildScript CheckExtractorRun(bool warnOnFailure) =>
-                BuildScript.Create(actions =>
-                {
-                    if (actions.FileExists(Extractor.GetCSharpLogPath()))
-                        return 0;
-
-                    if (warnOnFailure)
-                        Log(Severity.Error, "No C# code detected during build.");
-
-                    return 1;
-                });
-
             var attempt = BuildScript.Failure;
             switch (GetCSharpBuildStrategy())
             {
@@ -65,47 +58,9 @@ namespace Semmle.Autobuild.CSharp
                     attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true);
                     break;
                 case CSharpBuildStrategy.Auto:
-                    var cleanTrapFolder =
-                        BuildScript.DeleteDirectory(TrapDir);
-                    var cleanSourceArchive =
-                        BuildScript.DeleteDirectory(SourceArchiveDir);
-                    var tryCleanExtractorArgsLogs =
-                        BuildScript.Create(actions =>
-                        {
-                            foreach (var file in Extractor.GetCSharpArgsLogs())
-                            {
-                                try
-                                {
-                                    actions.FileDelete(file);
-                                }
-                                catch // lgtm[cs/catch-of-all-exceptions] lgtm[cs/empty-catch-block]
-                                { }
-                            }
-
-                            return 0;
-                        });
-                    var attemptExtractorCleanup =
-                        BuildScript.Try(cleanTrapFolder) &
-                        BuildScript.Try(cleanSourceArchive) &
-                        tryCleanExtractorArgsLogs &
-                        BuildScript.DeleteFile(Extractor.GetCSharpLogPath());
-
-                    /// <summary>
-                    /// Execute script `s` and check that the C# extractor has been executed.
-                    /// If either fails, attempt to cleanup any artifacts produced by the extractor,
-                    /// and exit with code 1, in order to proceed to the next attempt.
-                    /// </summary>
-                    BuildScript IntermediateAttempt(BuildScript s) =>
-                        (s & CheckExtractorRun(false)) |
-                        (attemptExtractorCleanup & BuildScript.Failure);
-
                     attempt =
-                        // First try .NET Core
-                        IntermediateAttempt(new DotNetRule().Analyse(this, true)) |
-                        // Then MSBuild
-                        (() => IntermediateAttempt(new MsBuildRule().Analyse(this, true))) |
-                        // And finally look for a script that might be a build script
-                        (() => new BuildCommandAutoRule(DotNetRule.WithDotNet).Analyse(this, true) & CheckExtractorRun(true)) |
+                        // Attempt a few different build strategies to see if one works
+                        this.autoBuildRule.Analyse(this, true) |
                         // All attempts failed: print message
                         AutobuildFailure();
                     break;
@@ -114,6 +69,127 @@ namespace Semmle.Autobuild.CSharp
             return attempt;
         }
 
+        /// <summary>
+        /// A script that checks that the C# extractor has been executed.
+        /// </summary>
+        public BuildScript CheckExtractorRun(bool warnOnFailure) =>
+            BuildScript.Create(actions =>
+            {
+                if (actions.FileExists(Extractor.GetCSharpLogPath()))
+                    return 0;
+
+                if (warnOnFailure)
+                    Log(Severity.Error, "No C# code detected during build.");
+
+                return 1;
+            });
+
+        protected override void AutobuildFailureDiagnostic()
+        {
+            // if `ScriptPath` is not null here, the `BuildCommandAuto` rule was
+            // run and found at least one script to execute
+            if (this.autoBuildRule.BuildCommandAutoRule.ScriptPath is not null)
+            {
+                var relScriptPath = this.MakeRelative(autoBuildRule.BuildCommandAutoRule.ScriptPath);
+
+                // if we found multiple build scripts in the project directory, then we can say
+                // as much to indicate that we may have picked the wrong one;
+                // otherwise, we just report that the one script we found didn't work
+                DiagnosticMessage message =
+                    this.autoBuildRule.BuildCommandAutoRule.CandidatePaths.Count() > 1 ?
+                    new(
+                        this.Options.Language,
+                        "multiple-build-scripts",
+                        "There are multiple potential build scripts",
+                        markdownMessage:
+                            "CodeQL found multiple potential build scripts for your project and " +
+                            $"attempted to run `{relScriptPath}`, which failed. " +
+                            "This may not be the right build script for your project. " +
+                            $"Set up a [manual build command]({buildCommandDocsUrl})."
+                    ) :
+                    new(
+                        this.Options.Language,
+                        "script-failure",
+                        "Unable to build project using build script",
+                        markdownMessage:
+                            "CodeQL attempted to build your project using a script located at " +
+                            $"`{relScriptPath}`, which failed. " +
+                            $"Set up a [manual build command]({buildCommandDocsUrl})."
+                    );
+
+                AddDiagnostic(message);
+            }
+
+            // project files which don't exist get marked as not .NET core projects, but we don't want
+            // to show an error for this if the files don't exist
+            var foundNotDotNetProjects = autoBuildRule.DotNetRule.NotDotNetProjects.Where(
+                proj => this.Actions.FileExists(proj.FullPath)
+            );
+
+            // both dotnet and msbuild builds require project or solution files; if we haven't found any
+            // then neither of those rules would've worked
+            if (this.ProjectsOrSolutionsToBuild.Count == 0)
+            {
+                this.AddDiagnostic(new(
+                    this.Options.Language,
+                    "no-projects-or-solutions",
+                    "No project or solutions files found",
+                    markdownMessage:
+                        "CodeQL could not find any project or solution files in your repository. " +
+                        $"Set up a [manual build command]({buildCommandDocsUrl})."
+                ));
+            }
+            // show a warning if there are projects which are not compatible with .NET Core, in case that is unintentional
+            else if (foundNotDotNetProjects.Any())
+            {
+                this.AddDiagnostic(new(
+                    this.Options.Language,
+                    "dotnet-incompatible-projects",
+                    "Some projects are incompatible with .NET Core",
+                    severity: DiagnosticMessage.TspSeverity.Warning,
+                    markdownMessage: $"""
+                    CodeQL found some projects which cannot be built with .NET Core:
+
+                    {autoBuildRule.DotNetRule.NotDotNetProjects.Select(p => this.MakeRelative(p.FullPath)).ToMarkdownList(MarkdownUtil.CodeFormatter, 5)}
+                    """
+                ));
+            }
+
+            // report any projects that failed to build with .NET Core
+            if (autoBuildRule.DotNetRule.FailedProjectsOrSolutions.Any())
+            {
+                this.AddDiagnostic(new(
+                    this.Options.Language,
+                    "dotnet-build-failure",
+                    "Some projects or solutions failed to build using .NET Core",
+                    markdownMessage: $"""
+                    CodeQL was unable to build the following projects using .NET Core:
+
+                    {autoBuildRule.DotNetRule.FailedProjectsOrSolutions.Select(p => this.MakeRelative(p.FullPath)).ToMarkdownList(MarkdownUtil.CodeFormatter, 10)}
+
+                    Set up a [manual build command]({buildCommandDocsUrl}).
+                    """
+                ));
+            }
+
+            // report any projects that failed to build with MSBuild
+            if (autoBuildRule.MsBuildRule.FailedProjectsOrSolutions.Any())
+            {
+                this.AddDiagnostic(new(
+                    this.Options.Language,
+                    "msbuild-build-failure",
+                    "Some projects or solutions failed to build using MSBuild",
+                    markdownMessage: $"""
+                    CodeQL was unable to build the following projects using MSBuild:
+
+                    {autoBuildRule.MsBuildRule.FailedProjectsOrSolutions.Select(p => this.MakeRelative(p.FullPath)).ToMarkdownList(MarkdownUtil.CodeFormatter, 10)}
+
+                    Set up a [manual build command]({buildCommandDocsUrl}).
+                    """
+                ));
+            }
+        }
+
         /// <summary>
         /// Gets the build strategy that the autobuilder should apply, based on the
         /// options in the `lgtm.yml` file.

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpDiagnosticClassifier.cs

@@ -0,0 +1,133 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Semmle.Autobuild.Shared;
+using Semmle.Util;
+
+namespace Semmle.Autobuild.CSharp
+{
+    /// <summary>
+    /// A diagnostic rule which tries to identify missing Xamarin SDKs.
+    /// </summary>
+    public class MissingXamarinSdkRule : DiagnosticRule
+    {
+        private const string docsUrl = "https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-xamarin-applications";
+
+        public class Result : IDiagnosticsResult
+        {
+            /// <summary>
+            /// The name of the SDK that is missing.
+            /// </summary>
+            public string SDKName { get; }
+
+            public Result(string sdkName)
+            {
+                this.SDKName = sdkName;
+            }
+
+            public DiagnosticMessage ToDiagnosticMessage<T>(Autobuilder<T> builder, DiagnosticMessage.TspSeverity? severity = null) where T : AutobuildOptionsShared => new(
+                builder.Options.Language,
+                $"missing-xamarin-{this.SDKName.ToLower()}-sdk",
+                $"Missing Xamarin SDK for {this.SDKName}",
+                severity: severity ?? DiagnosticMessage.TspSeverity.Error,
+                markdownMessage: $"[Configure your workflow]({docsUrl}) for this SDK before running CodeQL."
+            );
+        }
+
+        public MissingXamarinSdkRule() :
+            base("MSB4019:[^\"]*\"[^\"]*Xamarin\\.(?<sdkName>[^\\.]*)\\.CSharp\\.targets\"")
+        {
+        }
+
+        public override void Fire(DiagnosticClassifier classifier, Match match)
+        {
+            if (!match.Groups.TryGetValue("sdkName", out var sdkName))
+                throw new ArgumentException("Expected regular expression match to contain sdkName");
+
+            var xamarinResults = classifier.Results.OfType<Result>().Where(result =>
+                result.SDKName.Equals(sdkName.Value)
+            );
+
+            if (!xamarinResults.Any())
+                classifier.Results.Add(new Result(sdkName.Value));
+        }
+    }
+
+    public class MissingProjectFileRule : DiagnosticRule
+    {
+        private const string runsOnDocsUrl = "https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on";
+        private const string checkoutDocsUrl = "https://github.com/actions/checkout#usage";
+
+        public class Result : IDiagnosticsResult
+        {
+            /// <summary>
+            /// A set of missing project files.
+            /// </summary>
+            public HashSet<string> MissingProjectFiles { get; }
+
+            public Result()
+            {
+                this.MissingProjectFiles = new HashSet<string>();
+            }
+
+            public DiagnosticMessage ToDiagnosticMessage<T>(Autobuilder<T> builder, DiagnosticMessage.TspSeverity? severity = null) where T : AutobuildOptionsShared => new(
+                builder.Options.Language,
+                "missing-project-files",
+                "Missing project files",
+                severity: severity ?? DiagnosticMessage.TspSeverity.Warning,
+                markdownMessage: $"""
+                Some project files were not found when CodeQL built your project:
+
+                {this.MissingProjectFiles.AsEnumerable().Select(p => builder.MakeRelative(p)).ToMarkdownList(MarkdownUtil.CodeFormatter, 5)}
+
+                This may lead to subsequent failures. You can check for common causes for missing project files:
+
+                - Ensure that the project is built using the {runsOnDocsUrl.ToMarkdownLink("intended operating system")} and that filenames on case-sensitive platforms are correctly specified.
+                - If your repository uses Git submodules, ensure that those are {checkoutDocsUrl.ToMarkdownLink("checked out")} before the CodeQL action is run.
+                - If you auto-generate some project files as part of your build process, ensure that these are generated before the CodeQL action is run.
+                """
+            );
+        }
+
+        public MissingProjectFileRule() :
+            base("MSB3202: The project file \"(?<projectFile>[^\"]+)\" was not found. \\[(?<location>[^\\]]+)\\]")
+        {
+        }
+
+        public override void Fire(DiagnosticClassifier classifier, Match match)
+        {
+            if (!match.Groups.TryGetValue("projectFile", out var projectFile))
+                throw new ArgumentException("Expected regular expression match to contain projectFile");
+            if (!match.Groups.TryGetValue("location", out var location))
+                throw new ArgumentException("Expected regular expression match to contain location");
+
+            var result = classifier.Results.OfType<Result>().FirstOrDefault();
+
+            // if we do not yet have a result for this rule, create one and add it to the list
+            // of results the classifier knows about
+            if (result is null)
+            {
+                result = new Result();
+                classifier.Results.Add(result);
+            }
+
+            // then add the missing project file
+            result.MissingProjectFiles.Add(projectFile.Value);
+        }
+    }
+
+    /// <summary>
+    /// Implements a <see cref="DiagnosticClassifier" /> which applies C#-specific rules to
+    /// the build output.
+    /// </summary>
+    public class CSharpDiagnosticClassifier : DiagnosticClassifier
+    {
+        public CSharpDiagnosticClassifier()
+        {
+            // add C#-specific rules to this classifier
+            this.AddRule(new MissingXamarinSdkRule());
+            this.AddRule(new MissingProjectFileRule());
+        }
+    }
+}

csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs

@@ -15,6 +15,15 @@ namespace Semmle.Autobuild.CSharp
     /// </summary>
     internal class DotNetRule : IBuildRule<CSharpAutobuildOptions>
     {
+        public readonly List<IProjectOrSolution> FailedProjectsOrSolutions = new();
+
+        /// <summary>
+        /// A list of projects which are incompatible with DotNet.
+        /// </summary>
+        public IEnumerable<Project<CSharpAutobuildOptions>> NotDotNetProjects { get; private set; }
+
+        public DotNetRule() => NotDotNetProjects = new List<Project<CSharpAutobuildOptions>>();
+
         public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
         {
             if (!builder.ProjectsOrSolutionsToBuild.Any())
@@ -22,10 +31,12 @@ namespace Semmle.Autobuild.CSharp
 
             if (auto)
             {
-                var notDotNetProject = builder.ProjectsOrSolutionsToBuild
+                NotDotNetProjects = builder.ProjectsOrSolutionsToBuild
                     .SelectMany(p => Enumerators.Singleton(p).Concat(p.IncludedProjects))
                     .OfType<Project<CSharpAutobuildOptions>>()
-                    .FirstOrDefault(p => !p.DotNetProject);
+                    .Where(p => !p.DotNetProject);
+                var notDotNetProject = NotDotNetProjects.FirstOrDefault();
+
                 if (notDotNetProject is not null)
                 {
                     builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject);
@@ -50,7 +61,10 @@ namespace Semmle.Autobuild.CSharp
 
                         var build = GetBuildScript(builder, dotNetPath, environment, projectOrSolution.FullPath);
 
-                        ret &= BuildScript.Try(clean) & BuildScript.Try(restore) & build;
+                        ret &= BuildScript.Try(clean) & BuildScript.Try(restore) & BuildScript.OnFailure(build, ret =>
+                        {
+                            FailedProjectsOrSolutions.Add(projectOrSolution);
+                        });
                     }
                     return ret;
                 });

csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
+using Semmle.Util;
 
 namespace Semmle.Autobuild.Shared
 {

csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs

@@ -1,3 +1,4 @@
+using Semmle.Util;
 using Semmle.Util.Logging;
 using System;
 using System.Collections.Generic;
@@ -189,10 +190,11 @@ namespace Semmle.Autobuild.Shared
         /// solution file and tools.
         /// </summary>
         /// <param name="options">The command line options.</param>
-        protected Autobuilder(IBuildActions actions, TAutobuildOptions options)
+        protected Autobuilder(IBuildActions actions, TAutobuildOptions options, DiagnosticClassifier diagnosticClassifier)
         {
             Actions = actions;
             Options = options;
+            DiagnosticClassifier = diagnosticClassifier;
 
             pathsLazy = new Lazy<IEnumerable<(string, int)>>(() =>
             {
@@ -232,24 +234,53 @@ namespace Semmle.Autobuild.Shared
                 return ret ?? new List<IProjectOrSolution>();
             });
 
-            CodeQLExtractorLangRoot = Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_ROOT");
-            CodeQlPlatform = Actions.GetEnvironmentVariable("CODEQL_PLATFORM");
+            CodeQLExtractorLangRoot = Actions.GetEnvironmentVariable(EnvVars.Root(this.Options.Language));
+            CodeQlPlatform = Actions.GetEnvironmentVariable(EnvVars.Platform);
 
-            TrapDir =
-                Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_TRAP_DIR") ??
-                throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_TRAP_DIR has not been set.");
+            TrapDir = RequireEnvironmentVariable(EnvVars.TrapDir(this.Options.Language));
+            SourceArchiveDir = RequireEnvironmentVariable(EnvVars.SourceArchiveDir(this.Options.Language));
+            DiagnosticsDir = RequireEnvironmentVariable(EnvVars.DiagnosticDir(this.Options.Language));
 
-            SourceArchiveDir =
-                Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_SOURCE_ARCHIVE_DIR") ??
-                throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_SOURCE_ARCHIVE_DIR has not been set.");
+            this.diagnostics = actions.CreateDiagnosticsWriter(Path.Combine(DiagnosticsDir, $"autobuilder-{DateTime.UtcNow:yyyyMMddHHmm}.jsonc"));
         }
 
-        protected string TrapDir { get; }
+        /// <summary>
+        /// Retrieves the value of an environment variable named <paramref name="name"> or throws
+        /// an exception if no such environment variable has been set.
+        /// </summary>
+        /// <param name="name">The name of the environment variable.</param>
+        /// <returns>The value of the environment variable.</returns>
+        /// <exception cref="InvalidEnvironmentException">
+        /// Thrown if the environment variable is not set.
+        /// </exception>
+        protected string RequireEnvironmentVariable(string name)
+        {
+            return Actions.GetEnvironmentVariable(name) ??
+                throw new InvalidEnvironmentException($"The environment variable {name} has not been set.");
+        }
 
-        protected string SourceArchiveDir { get; }
+        public string TrapDir { get; }
+
+        public string SourceArchiveDir { get; }
+
+        public string DiagnosticsDir { get; }
+
+        protected DiagnosticClassifier DiagnosticClassifier { get; }
 
         private readonly ILogger logger = new ConsoleLogger(Verbosity.Info);
 
+        private readonly IDiagnosticsWriter diagnostics;
+
+        /// <summary>
+        /// Makes <see cref="path" /> relative to the root source directory.
+        /// </summary>
+        /// <param name="path">The path which to make relative.</param>
+        /// <returns>The relative path.</returns>
+        public string MakeRelative(string path)
+        {
+            return Path.GetRelativePath(this.RootDirectory, path);
+        }
+
         /// <summary>
         /// Log a given build event to the console.
         /// </summary>
@@ -260,6 +291,15 @@ namespace Semmle.Autobuild.Shared
             logger.Log(severity, format, args);
         }
 
+        /// <summary>
+        /// Write <paramref name="diagnostic"/> to the diagnostics file.
+        /// </summary>
+        /// <param name="diagnostic">The diagnostics entry to write.</param>
+        public void AddDiagnostic(DiagnosticMessage diagnostic)
+        {
+            diagnostics.AddEntry(diagnostic);
+        }
+
         /// <summary>
         /// Attempt to build this project.
         /// </summary>
@@ -283,7 +323,19 @@ namespace Semmle.Autobuild.Shared
                 Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
             }
 
-            return script.Run(Actions, startCallback, exitCallback);
+            var onOutput = BuildOutputHandler(Console.Out);
+            var onError = BuildOutputHandler(Console.Error);
+
+            var buildResult = script.Run(Actions, startCallback, exitCallback, onOutput, onError);
+
+            // if the build succeeded, all diagnostics we captured from the build output should be warnings;
+            // otherwise they should all be errors
+            var diagSeverity = buildResult == 0 ? DiagnosticMessage.TspSeverity.Warning : DiagnosticMessage.TspSeverity.Error;
+            this.DiagnosticClassifier.Results
+                .Select(result => result.ToDiagnosticMessage(this, diagSeverity))
+                .ForEach(AddDiagnostic);
+
+            return buildResult;
         }
 
         /// <summary>
@@ -291,13 +343,58 @@ namespace Semmle.Autobuild.Shared
         /// </summary>
         public abstract BuildScript GetBuildScript();
 
+
+        /// <summary>
+        /// Produces a diagnostic for the tool status page that we were unable to automatically
+        /// build the user's project and that they can manually specify a build command. This
+        /// can be overriden to implement more specific messages depending on the origin of
+        /// the failure.
+        /// </summary>
+        protected virtual void AutobuildFailureDiagnostic() => AddDiagnostic(new DiagnosticMessage(
+                this.Options.Language,
+                "autobuild-failure",
+                "Unable to build project",
+                visibility: new DiagnosticMessage.TspVisibility(statusPage: true),
+                plaintextMessage: """
+                    We were unable to automatically build your project.
+                    Set up a manual build command.
+                """
+            ));
+
+        /// <summary>
+        /// Returns a build script that can be run upon autobuild failure.
+        /// </summary>
+        /// <returns>
+        /// A build script that reports that we could not automatically detect a suitable build method.
+        /// </returns>
         protected BuildScript AutobuildFailure() =>
             BuildScript.Create(actions =>
                 {
                     Log(Severity.Error, "Could not auto-detect a suitable build method");
+
+                    AutobuildFailureDiagnostic();
+
                     return 1;
                 });
 
+        /// <summary>
+        /// Constructs a <see cref="BuildOutputHandler" /> which uses the <see cref="DiagnosticClassifier" />
+        /// to classify build output. All data also gets written to <paramref name="writer" />.
+        /// </summary>
+        /// <param name="writer">
+        /// The <see cref="TextWriter" /> to which the build output would have normally been written to.
+        /// This is normally <see cref="Console.Out" /> or <see cref="Console.Error" />.
+        /// </param>
+        /// <returns>The constructed <see cref="BuildOutputHandler" />.</returns>
+        protected BuildOutputHandler BuildOutputHandler(TextWriter writer) => new(data =>
+        {
+            if (data is not null)
+            {
+                writer.WriteLine(data);
+                DiagnosticClassifier.ClassifyLine(data);
+            }
+        });
+
         /// <summary>
         /// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
         /// </summary>

csharp/autobuilder/Semmle.Autobuild.Shared/BuildActions.cs

@@ -7,14 +7,30 @@ using System.Xml;
 using System.Net.Http;
 using System.Diagnostics.CodeAnalysis;
 using System.Threading.Tasks;
+using System.Runtime.InteropServices;
 
 namespace Semmle.Autobuild.Shared
 {
+    public delegate void BuildOutputHandler(string? data);
+
     /// <summary>
     /// Wrapper around system calls so that the build scripts can be unit-tested.
     /// </summary>
     public interface IBuildActions
     {
+
+        /// <summary>
+        /// Runs a process, captures its output, and provides it asynchronously.
+        /// </summary>
+        /// <param name="exe">The exe to run.</param>
+        /// <param name="args">The other command line arguments.</param>
+        /// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
+        /// <param name="env">Additional environment variables.</param>
+        /// <param name="onOutput">A handler for stdout output.</param>
+        /// <param name="onError">A handler for stderr output.</param>
+        /// <returns>The process exit code.</returns>
+        int RunProcess(string exe, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError);
+
         /// <summary>
         /// Runs a process and captures its output.
         /// </summary>
@@ -98,6 +114,18 @@ namespace Semmle.Autobuild.Shared
         /// </summary>
         bool IsWindows();
 
+        /// <summary>
+        /// Gets a value indicating whether we are running on macOS.
+        /// </summary>
+        /// <returns>True if we are running on macOS.</returns>
+        bool IsMacOs();
+
+        /// <summary>
+        /// Gets a value indicating whether we are running on arm.
+        /// </summary>
+        /// <returns>True if we are running on arm.</returns>
+        bool IsArm();
+
         /// <summary>
         /// Combine path segments, Path.Combine().
         /// </summary>
@@ -139,6 +167,17 @@ namespace Semmle.Autobuild.Shared
         /// Downloads the resource with the specified URI to a local file.
         /// </summary>
         void DownloadFile(string address, string fileName);
+
+        /// <summary>
+        /// Creates an <see cref="IDiagnosticsWriter" /> for the given <paramref name="filename" />.
+        /// </summary>
+        /// <param name="filename">
+        /// The path suggesting where the diagnostics should be written to.
+        /// </param>
+        /// <returns>
+        /// A <see cref="IDiagnosticsWriter" /> to which diagnostic entries can be added.
+        /// </returns>
+        IDiagnosticsWriter CreateDiagnosticsWriter(string filename);
     }
 
     /// <summary>
@@ -169,6 +208,26 @@ namespace Semmle.Autobuild.Shared
             return pi;
         }
 
+        int IBuildActions.RunProcess(string exe, string args, string? workingDirectory, System.Collections.Generic.IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
+        {
+            var pi = GetProcessStartInfo(exe, args, workingDirectory, env, true);
+            using var p = new Process
+            {
+                StartInfo = pi
+            };
+            p.StartInfo.RedirectStandardError = true;
+            p.OutputDataReceived += new DataReceivedEventHandler((sender, e) => onOutput(e.Data));
+            p.ErrorDataReceived += new DataReceivedEventHandler((sender, e) => onError(e.Data));
+
+            p.Start();
+
+            p.BeginErrorReadLine();
+            p.BeginOutputReadLine();
+
+            p.WaitForExit();
+            return p.ExitCode;
+        }
+
         int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? environment)
         {
             var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment, false);
@@ -203,6 +262,12 @@ namespace Semmle.Autobuild.Shared
 
         bool IBuildActions.IsWindows() => Win32.IsWindows();
 
+        bool IBuildActions.IsMacOs() => RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
+
+        bool IBuildActions.IsArm() =>
+            RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ||
+            RuntimeInformation.ProcessArchitecture == Architecture.Arm;
+
         string IBuildActions.PathCombine(params string[] parts) => Path.Combine(parts);
 
         void IBuildActions.WriteAllText(string filename, string contents) => File.WriteAllText(filename, contents);
@@ -234,6 +299,8 @@ namespace Semmle.Autobuild.Shared
         public void DownloadFile(string address, string fileName) =>
             DownloadFileAsync(address, fileName).Wait();
 
+        public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new DiagnosticsStream(filename);
+
         public static IBuildActions Instance { get; } = new SystemBuildActions();
     }
 }

csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandAutoRule.cs

@@ -11,22 +11,42 @@ namespace Semmle.Autobuild.Shared
     {
         private readonly WithDotNet<AutobuildOptionsShared> withDotNet;
 
+        /// <summary>
+        /// A list of paths to files in the project directory which we classified as scripts.
+        /// </summary>
+        public IEnumerable<string> CandidatePaths { get; private set; }
+
+        /// <summary>
+        /// The path of the script we decided to run, if any.
+        /// </summary>
+        public string? ScriptPath { get; private set; }
+
         public BuildCommandAutoRule(WithDotNet<AutobuildOptionsShared> withDotNet)
         {
             this.withDotNet = withDotNet;
+            this.CandidatePaths = new List<string>();
         }
 
+        /// <summary>
+        /// A list of extensions that we consider to be for scripts on Windows.
+        /// </summary>
         private readonly IEnumerable<string> winExtensions = new List<string> {
             ".bat",
             ".cmd",
             ".exe"
         };
 
+        /// <summary>
+        /// A list of extensions that we consider to be for scripts on Linux.
+        /// </summary>
         private readonly IEnumerable<string> linuxExtensions = new List<string> {
             "",
             ".sh"
         };
 
+        /// <summary>
+        /// A list of filenames without extensions that we think might be build scripts.
+        /// </summary>
         private readonly IEnumerable<string> buildScripts = new List<string> {
             "build"
         };
@@ -35,18 +55,25 @@ namespace Semmle.Autobuild.Shared
         {
             builder.Log(Severity.Info, "Attempting to locate build script");
 
+            // a list of extensions for files that we consider to be scripts on the current platform
             var extensions = builder.Actions.IsWindows() ? winExtensions : linuxExtensions;
+            // a list of combined base script names with the current platform's script extensions
+            // e.g. for Linux: build, build.sh
             var scripts = buildScripts.SelectMany(s => extensions.Select(e => s + e));
-            var scriptPath = builder.Paths.Where(p => scripts.Any(p.Item1.ToLower().EndsWith)).OrderBy(p => p.Item2).Select(p => p.Item1).FirstOrDefault();
+            // search through the files in the project directory for paths which end in one of
+            // the names given by `scripts`, then order them by their distance from the root
+            this.CandidatePaths = builder.Paths.Where(p => scripts.Any(p.Item1.ToLower().EndsWith)).OrderBy(p => p.Item2).Select(p => p.Item1);
+            // pick the first matching path, if there is one
+            this.ScriptPath = this.CandidatePaths.FirstOrDefault();
 
-            if (scriptPath is null)
+            if (this.ScriptPath is null)
                 return BuildScript.Failure;
 
             var chmod = new CommandBuilder(builder.Actions);
-            chmod.RunCommand("/bin/chmod", $"u+x {scriptPath}");
+            chmod.RunCommand("/bin/chmod", $"u+x {this.ScriptPath}");
             var chmodScript = builder.Actions.IsWindows() ? BuildScript.Success : BuildScript.Try(chmod.Script);
 
-            var dir = builder.Actions.GetDirectoryName(scriptPath);
+            var dir = builder.Actions.GetDirectoryName(this.ScriptPath);
 
             // A specific .NET Core version may be required
             return chmodScript & withDotNet(builder, environment =>
@@ -58,7 +85,7 @@ namespace Semmle.Autobuild.Shared
                 if (vsTools is not null)
                     command.CallBatFile(vsTools.Path);
 
-                command.RunCommand(scriptPath);
+                command.RunCommand(this.ScriptPath);
                 return command.Script;
             });
         }

csharp/autobuilder/Semmle.Autobuild.Shared/BuildScript.cs

@@ -46,6 +46,33 @@ namespace Semmle.Autobuild.Shared
         /// <returns>The exit code from this build script.</returns>
         public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout);
 
+        /// <summary>
+        /// Runs this build command.
+        /// </summary>
+        /// <param name="actions">
+        /// The interface used to implement the build actions.
+        /// </param>
+        /// <param name="startCallback">
+        /// A call back that is called every time a new process is started. The
+        /// argument to the call back is a textual representation of the process.
+        /// </param>
+        /// <param name="exitCallBack">
+        /// A call back that is called every time a new process exits. The first
+        /// argument to the call back is the exit code, and the second argument is
+        /// an exit message.
+        /// </param>
+        /// <param name="onOutput">
+        /// A handler for data read from stdout.
+        /// </param>
+        /// <param name="onError">
+        /// A handler for data read from stderr.
+        /// </param>
+        /// <returns>The exit code from this build script.</returns>
+        public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError);
+
+        /// <summary>
+        /// A build script which executes an external program or script.
+        /// </summary>
         private class BuildCommand : BuildScript
         {
             private readonly string exe, arguments;
@@ -110,8 +137,29 @@ namespace Semmle.Autobuild.Shared
                 return ret;
             }
 
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError)
+            {
+                startCallback(this.ToString(), silent);
+                var ret = 1;
+                var retMessage = "";
+                try
+                {
+                    ret = actions.RunProcess(exe, arguments, workingDirectory, environment, onOutput, onError);
+                }
+                catch (Exception ex)
+                    when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException)
+                {
+                    retMessage = ex.Message;
+                }
+                exitCallBack(ret, retMessage, silent);
+                return ret;
+            }
+
         }
 
+        /// <summary>
+        /// A build script which runs a C# function.
+        /// </summary>
         private class ReturnBuildCommand : BuildScript
         {
             private readonly Func<IBuildActions, int> func;
@@ -127,8 +175,13 @@ namespace Semmle.Autobuild.Shared
                 stdout = Array.Empty<string>();
                 return func(actions);
             }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError) => func(actions);
         }
 
+        /// <summary>
+        /// Allows two build scripts to be composed sequentially.
+        /// </summary>
         private class BindBuildScript : BuildScript
         {
             private readonly BuildScript s1;
@@ -175,6 +228,32 @@ namespace Semmle.Autobuild.Shared
                 stdout = @out;
                 return ret2;
             }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError)
+            {
+                int ret1;
+                if (s2a is not null)
+                {
+                    var stdout1 = new List<string>();
+                    var onOutputWrapper = new BuildOutputHandler(data =>
+                    {
+                        if (data is not null)
+                            stdout1.Add(data);
+
+                        onOutput(data);
+                    });
+                    ret1 = s1.Run(actions, startCallback, exitCallBack, onOutputWrapper, onError);
+                    return s2a(stdout1, ret1).Run(actions, startCallback, exitCallBack, onOutput, onError);
+                }
+
+                if (s2b is not null)
+                {
+                    ret1 = s1.Run(actions, startCallback, exitCallBack, onOutput, onError);
+                    return s2b(ret1).Run(actions, startCallback, exitCallBack, onOutput, onError);
+                }
+
+                throw new InvalidOperationException("Unexpected error");
+            }
         }
 
         /// <summary>
@@ -260,6 +339,23 @@ namespace Semmle.Autobuild.Shared
         /// </summary>
         public static BuildScript Try(BuildScript s) => s | Success;
 
+        /// <summary>
+        /// Creates a build script that runs the build script <paramref name="s" />. If
+        /// running <paramref name="s" /> fails, <paramref name="k" /> is invoked with
+        /// the exit code.
+        /// </summary>
+        /// <param name="s">The build script to run.</param>
+        /// <param name="k">
+        /// The callback that is invoked if <paramref name="s" /> failed.
+        /// </param>
+        /// <returns>The build script which implements this.</returns>
+        public static BuildScript OnFailure(BuildScript s, Action<int> k) =>
+            new BindBuildScript(s, ret => Create(actions =>
+            {
+                if (!Succeeded(ret)) k(ret);
+                return ret;
+            }));
+
         /// <summary>
         /// Creates a build script that deletes the given directory.
         /// </summary>

csharp/autobuilder/Semmle.Autobuild.Shared/DiagnosticClassifier.cs

@@ -0,0 +1,206 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Semmle.Util;
+
+namespace Semmle.Autobuild.Shared
+{
+    /// <summary>
+    /// Direct results result from the successful application of a <see cref="DiagnosticRule" />,
+    /// which can later be converted to a corresponding <see cref="DiagnosticMessage" />.
+    /// </summary>
+    public interface IDiagnosticsResult
+    {
+        /// <summary>
+        /// Produces a <see cref="DiagnosticMessage" /> corresponding to this result.
+        /// </summary>
+        /// <param name="builder">
+        /// The autobuilder to use for constructing the base <see cref="DiagnosticMessage" />.
+        /// </param>
+        /// <param name="severity">
+        /// An optional severity value which overrides the default severity of the diagnostic.
+        /// </param>
+        /// <returns>The <see cref="DiagnosticMessage" /> corresponding to this result.</returns>
+        DiagnosticMessage ToDiagnosticMessage<T>(Autobuilder<T> builder, DiagnosticMessage.TspSeverity? severity = null) where T : AutobuildOptionsShared;
+    }
+
+    public class DiagnosticRule
+    {
+        /// <summary>
+        /// The pattern against which this rule matches build output.
+        /// </summary>
+        public Regex Pattern { get; }
+
+        /// <summary>
+        /// Constructs a diagnostic rule for the given <paramref name="pattern" />.
+        /// </summary>
+        /// <param name="pattern"></param>
+        public DiagnosticRule(Regex pattern)
+        {
+            this.Pattern = pattern;
+        }
+
+        /// <summary>
+        /// Constructs a diagnostic rule for the given regular expression <paramref name="pattern" />.
+        /// </summary>
+        /// <param name="pattern"></param>
+        public DiagnosticRule(string pattern)
+        {
+            this.Pattern = new Regex(pattern, RegexOptions.Compiled);
+        }
+
+        /// <summary>
+        /// Used by a <see cref="DiagnosticClassifier" /> <paramref name="classifier" /> to
+        /// signal that the rule has matched some build output with <paramref name="match" />.
+        /// </summary>
+        /// <param name="classifier">The classifier which is firing the rule.</param>
+        /// <param name="match">The <see cref="Match" /> that resulted from applying the rule.</param>
+        public virtual void Fire(DiagnosticClassifier classifier, Match match) { }
+    }
+
+    /// <summary>
+    /// A <see cref="DiagnosticRule" /> which detects arbitrary build messages from MSBuild, CSC, NETSDK, etc. and
+    /// reports them, up to a limit, as telemetry-only diagnostics.
+    /// </summary>
+    public class BuildMessageRule : DiagnosticRule
+    {
+        public class Result : IDiagnosticsResult, IEquatable<Result>
+        {
+            /// <summary>
+            /// A value indicating whether this a warning or an error.
+            /// </summary>
+            public string Type { get; }
+            /// <summary>
+            /// The source of the message, such as "MSB" or "NETSDK".
+            /// </summary>
+            public string Source { get; }
+            /// <summary>
+            /// The numeric id of the message.
+            /// </summary>
+            public int Code { get; }
+            /// <summary>
+            /// The message contents.
+            /// </summary>
+            public string Message { get; }
+
+            public Result(string type, string source, int code, string message)
+            {
+                this.Type = type;
+                this.Source = source;
+                this.Code = code;
+                this.Message = message;
+            }
+
+            public DiagnosticMessage ToDiagnosticMessage<T>(Autobuilder<T> builder, DiagnosticMessage.TspSeverity? severity = null) where T : AutobuildOptionsShared => new(
+                builder.Options.Language,
+                $"{this.Source.ToLower()}-{this.Code}",
+                $"{this.Source.ToUpper()}{this.Code}",
+                plaintextMessage: this.Message,
+                severity:
+                    this.Type.Equals("error") ?
+                    DiagnosticMessage.TspSeverity.Error :
+                    DiagnosticMessage.TspSeverity.Warning,
+                // the messages we capture here are visible in the build log, so there is no need
+                // to show them other than in telemetry
+                visibility: new(telemetry: true)
+            );
+
+            public bool Equals(Result? x)
+            {
+                return x is not null &&
+                    x.Code == this.Code &&
+                    x.Type == this.Type &&
+                    x.Source == this.Source &&
+                    x.Message == this.Message;
+            }
+
+            public override bool Equals(object? obj)
+            {
+                return obj is Result && this.Equals(obj);
+            }
+
+            public override int GetHashCode()
+            {
+                return HashCode.Combine(this.Type, this.Source, this.Code, this.Message);
+            }
+        }
+
+        /// <summary>
+        /// The maximum number of diagnostics we should emit for this rule.
+        /// </summary>
+        private const int maxDiagnostics = 10;
+        /// <summary>
+        /// The number of diagnostics this rule has emitted so far.
+        /// </summary>
+        private int diagnostics = 0;
+
+        public BuildMessageRule() : base("(?<type>error|warning) (?<source>[A-Z]+)(?<code>\\d+): (?<message>.*)") { }
+
+        public override void Fire(DiagnosticClassifier classifier, Match match)
+        {
+            if (!match.Groups.TryGetValue("type", out var type))
+                throw new ArgumentException("Expected regular expression match to contain type");
+            if (!match.Groups.TryGetValue("source", out var source))
+                throw new ArgumentException("Expected regular expression match to contain source");
+            if (!match.Groups.TryGetValue("code", out var codeStr))
+                throw new ArgumentException("Expected regular expression match to contain code");
+            if (!match.Groups.TryGetValue("message", out var message))
+                throw new ArgumentException("Expected regular expression match to contain message");
+            if (!int.TryParse(codeStr.Value, out var code))
+                throw new ArgumentException("Expected code to be numeric");
+
+            // check that we have not yet exceeded our limit for emitting diagnostics
+            if (this.diagnostics < BuildMessageRule.maxDiagnostics)
+            {
+                var result = new Result(type.Value, source.Value, code, message.Value);
+
+                // add this result if we don't already have an identical one
+                if (!classifier.Results.OfType<Result>().Any(d => d.Equals(result)))
+                {
+                    classifier.Results.Add(result);
+                    this.diagnostics++;
+                }
+            }
+        }
+    }
+
+    public class DiagnosticClassifier
+    {
+        private readonly List<DiagnosticRule> rules;
+        public readonly List<IDiagnosticsResult> Results;
+
+        public DiagnosticClassifier()
+        {
+            this.rules = new List<DiagnosticRule>();
+            this.Results = new List<IDiagnosticsResult>();
+
+            this.AddRule(new BuildMessageRule());
+        }
+
+        /// <summary>
+        /// Adds <paramref name="rule" /> to this classifier.
+        /// </summary>
+        /// <param name="rule">The rule to add.</param>
+        protected void AddRule(DiagnosticRule rule)
+        {
+            this.rules.Add(rule);
+        }
+
+        /// <summary>
+        /// Applies all of this classifier's rules to <paramref name="line" /> to see which match.
+        /// </summary>
+        /// <param name="line">The line to which the rules should be applied to.</param>
+        public void ClassifyLine(string line)
+        {
+            this.rules.ForEach(rule =>
+            {
+                var match = rule.Pattern.Match(line);
+                if (match.Success)
+                {
+                    rule.Fire(this, match);
+                }
+            });
+        }
+    }
+}

csharp/autobuilder/Semmle.Autobuild.Shared/EnvVars.cs

@@ -0,0 +1,13 @@
+using Semmle.Util;
+
+namespace Semmle.Autobuild.Shared
+{
+    public static class EnvVars
+    {
+        public const string Platform = "CODEQL_PLATFORM";
+        public static string Root(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_ROOT";
+        public static string TrapDir(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_TRAP_DIR";
+        public static string SourceArchiveDir(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_SOURCE_ARCHIVE_DIR";
+        public static string DiagnosticDir(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_DIAGNOSTIC_DIR";
+    }
+}

csharp/autobuilder/Semmle.Autobuild.Shared/MarkdownUtil.cs

@@ -0,0 +1,65 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Semmle.Autobuild.Shared
+{
+    public static class MarkdownUtil
+    {
+        /// <summary>
+        /// Formats items as markdown inline code.
+        /// </summary>
+        /// <returns>A function which formats items as markdown inline code.</returns>
+        public static readonly Func<string, string> CodeFormatter = item => $"`{item}`";
+
+        /// <summary>
+        /// Formats the string as a markdown link.
+        /// </summary>
+        /// <param name="link">The URL for the link.</param>
+        /// <param name="title">The text that is displayed.</param>
+        /// <returns>A string containing a markdown-formatted link.</returns>
+        public static string ToMarkdownLink(this string link, string title) => $"[{title}]({link})";
+
+        /// <summary>
+        /// Renders <see cref="projects" /> as a markdown list of the project paths.
+        /// </summary>
+        /// <param name="projects">
+        /// The list of projects whose paths should be rendered as a markdown list.
+        /// </param>
+        /// <param name="limit">The maximum number of items to include in the list.</param>
+        /// <returns>Returns the markdown list as a string.</returns>
+        public static string ToMarkdownList(this IEnumerable<IProjectOrSolution> projects, int? limit = null)
+        {
+            return projects.ToMarkdownList(p => $"`{p.FullPath}`", limit);
+        }
+
+        /// <summary>
+        /// Renders <see cref="items" /> as a markdown list.
+        /// </summary>
+        /// <typeparam name="T">The item type.</typeparam>
+        /// <param name="items">The list that should be formatted as a markdown list.</param>
+        /// <param name="formatter">A function which converts individual items into a string representation.</param>
+        /// <param name="limit">The maximum number of items to include in the list.</param>
+        /// <returns>Returns the markdown list as a string.</returns>
+        public static string ToMarkdownList<T>(this IEnumerable<T> items, Func<T, string> formatter, int? limit = null)
+        {
+            var sb = new StringBuilder();
+
+            // if there is a limit, take at most that many items from the  start of the list
+            var list = limit is not null ? items.Take(limit.Value) : items;
+            sb.Append(string.Join('\n', list.Select(item => $"- {formatter(item)}")));
+
+            // if there were more items than allowed in the list, add an extra item indicating
+            // how many more items there were
+            var length = items.Count();
+
+            if (limit is not null && length > limit)
+            {
+                sb.Append($"\n- and {length - limit} more. View the CodeQL logs for a full list.");
+            }
+
+            return sb.ToString();
+        }
+    }
+}

csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs

@@ -1,17 +1,39 @@
 using Semmle.Util.Logging;
+using System.Collections.Generic;
 using System.Linq;
 
 namespace Semmle.Autobuild.Shared
 {
+    internal static class MsBuildCommandExtensions
+    {
+        /// <summary>
+        /// Appends a call to msbuild.
+        /// </summary>
+        /// <param name="cmdBuilder"></param>
+        /// <param name="builder"></param>
+        /// <returns></returns>
+        public static CommandBuilder MsBuildCommand(this CommandBuilder cmdBuilder, IAutobuilder<AutobuildOptionsShared> builder)
+        {
+            var isArmMac = builder.Actions.IsMacOs() && builder.Actions.IsArm();
+
+            // mono doesn't ship with `msbuild` on Arm-based Macs, but we can fall back to
+            // msbuild that ships with `dotnet` which can be invoked with `dotnet msbuild`
+            // perhaps we should do this on all platforms?
+            return isArmMac ?
+                cmdBuilder.RunCommand("dotnet").Argument("msbuild") :
+                cmdBuilder.RunCommand("msbuild");
+        }
+    }
+
     /// <summary>
     /// A build rule using msbuild.
     /// </summary>
     public class MsBuildRule : IBuildRule<AutobuildOptionsShared>
     {
         /// <summary>
-        /// The name of the msbuild command.
+        /// A list of solutions or projects which failed to build.
         /// </summary>
-        private const string msBuild = "msbuild";
+        public readonly List<IProjectOrSolution> FailedProjectsOrSolutions = new();
 
         public BuildScript Analyse(IAutobuilder<AutobuildOptionsShared> builder, bool auto)
         {
@@ -57,7 +79,7 @@ namespace Semmle.Autobuild.Shared
                             Script;
                     var nugetRestore = GetNugetRestoreScript();
                     var msbuildRestoreCommand = new CommandBuilder(builder.Actions).
-                        RunCommand(msBuild).
+                        MsBuildCommand(builder).
                         Argument("/t:restore").
                         QuoteArgument(projectOrSolution.FullPath);
 
@@ -95,7 +117,7 @@ namespace Semmle.Autobuild.Shared
                     command.RunCommand("set Platform=&& type NUL", quoteExe: false);
                 }
 
-                command.RunCommand(msBuild);
+                command.MsBuildCommand(builder);
                 command.QuoteArgument(projectOrSolution.FullPath);
 
                 var target = builder.Options.MsBuildTarget ?? "rebuild";
@@ -110,7 +132,13 @@ namespace Semmle.Autobuild.Shared
 
                 command.Argument(builder.Options.MsBuildArguments);
 
-                ret &= command.Script;
+                // append the build script which invokes msbuild to the overall build script `ret`;
+                // we insert a check that building the current project or solution was successful:
+                // if it was not successful, we add it to `FailedProjectsOrSolutions`
+                ret &= BuildScript.OnFailure(command.Script, ret =>
+                {
+                    FailedProjectsOrSolutions.Add(projectOrSolution);
+                });
             }
 
             return ret;

csharp/autobuilder/Semmle.Autobuild.Shared/ProjectOrSolution.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using Semmle.Util;
 
 namespace Semmle.Autobuild.Shared
 {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Conversion.cs

@@ -10,6 +10,8 @@ namespace Semmle.Extraction.CSharp.Entities
         private Conversion(Context cx, IMethodSymbol init)
             : base(cx, init) { }
 
+        protected override MethodKind ExplicitlyImplementsKind => MethodKind.Conversion;
+
         public static new Conversion Create(Context cx, IMethodSymbol symbol) =>
             ConversionFactory.Instance.CreateEntityFromSymbol(cx, symbol);
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs

@@ -83,10 +83,12 @@ namespace Semmle.Extraction.CSharp.Entities
             }
         }
 
+        protected virtual MethodKind ExplicitlyImplementsKind => MethodKind.Ordinary;
+
         public void Overrides(TextWriter trapFile)
         {
             foreach (var explicitInterface in Symbol.ExplicitInterfaceImplementations
-                .Where(sym => sym.MethodKind == MethodKind.Ordinary)
+                .Where(sym => sym.MethodKind == ExplicitlyImplementsKind)
                 .Select(impl => Type.Create(Context, impl.ContainingType)))
             {
                 trapFile.explicitly_implements(this, explicitInterface.TypeRef);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs

@@ -65,6 +65,15 @@ namespace Semmle.Extraction.CSharp.Entities
             trapFile.has_modifiers(target, Modifier.Create(cx, modifier));
         }
 
+        private static void ExtractFieldModifiers(Context cx, TextWriter trapFile, IEntity key, IFieldSymbol symbol)
+        {
+            if (symbol.IsReadOnly)
+                HasModifier(cx, trapFile, key, Modifiers.Readonly);
+
+            if (symbol.IsRequired)
+                HasModifier(cx, trapFile, key, Modifiers.Required);
+        }
+
         private static void ExtractNamedTypeModifiers(Context cx, TextWriter trapFile, IEntity key, ISymbol symbol)
         {
             if (symbol.Kind != SymbolKind.NamedType)
@@ -76,6 +85,9 @@ namespace Semmle.Extraction.CSharp.Entities
             if (nt.IsRecord)
                 HasModifier(cx, trapFile, key, Modifiers.Record);
 
+            if (nt.IsFileLocal)
+                HasModifier(cx, trapFile, key, Modifiers.File);
+
             if (nt.TypeKind == TypeKind.Struct)
             {
                 if (nt.IsReadOnly)
@@ -88,7 +100,11 @@ namespace Semmle.Extraction.CSharp.Entities
 
         public static void ExtractModifiers(Context cx, TextWriter trapFile, IEntity key, ISymbol symbol)
         {
-            HasAccessibility(cx, trapFile, key, symbol.DeclaredAccessibility);
+            // A file scoped type has declared accessibility `internal` which we shouldn't extract.
+            // The file modifier is extracted as a source level modifier.
+            if (symbol.Kind != SymbolKind.NamedType || !((INamedTypeSymbol)symbol).IsFileLocal)
+                HasAccessibility(cx, trapFile, key, symbol.DeclaredAccessibility);
+
             if (symbol.Kind == SymbolKind.ErrorType)
                 trapFile.has_modifiers(key, Modifier.Create(cx, Accessibility.Public));
 
@@ -106,8 +122,11 @@ namespace Semmle.Extraction.CSharp.Entities
             if (symbol.IsVirtual)
                 HasModifier(cx, trapFile, key, Modifiers.Virtual);
 
-            if (symbol.Kind == SymbolKind.Field && ((IFieldSymbol)symbol).IsReadOnly)
-                HasModifier(cx, trapFile, key, Modifiers.Readonly);
+            if (symbol is IFieldSymbol field)
+                ExtractFieldModifiers(cx, trapFile, key, field);
+
+            if (symbol.Kind == SymbolKind.Property && ((IPropertySymbol)symbol).IsRequired)
+                HasModifier(cx, trapFile, key, Modifiers.Required);
 
             if (symbol.IsOverride)
                 HasModifier(cx, trapFile, key, Modifiers.Override);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifiers.cs

@@ -4,6 +4,7 @@ internal static class Modifiers
     public const string Async = "async";
     public const string Const = "const";
     public const string Extern = "extern";
+    public const string File = "file";
     public const string Internal = "internal";
     public const string New = "new";
     public const string Override = "override";
@@ -13,6 +14,7 @@ internal static class Modifiers
     public const string Public = "public";
     public const string Readonly = "readonly";
     public const string Record = "record";
+    public const string Required = "required";
     public const string Ref = "ref";
     public const string Sealed = "sealed";
     public const string Static = "static";

csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs

@@ -11,6 +11,8 @@ namespace Semmle.Extraction.CSharp.Entities
         protected UserOperator(Context cx, IMethodSymbol init)
             : base(cx, init) { }
 
+        protected override MethodKind ExplicitlyImplementsKind => MethodKind.UserDefinedOperator;
+
         public override void Populate(TextWriter trapFile)
         {
             PopulateMethod(trapFile);
@@ -37,6 +39,7 @@ namespace Semmle.Extraction.CSharp.Entities
             }
 
             ContainingType.PopulateGenerics();
+            Overrides(trapFile);
         }
 
         public override bool NeedsPopulation => Context.Defines(Symbol) || IsImplicitOperator(out _);

csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs

@@ -77,12 +77,8 @@ namespace Semmle.Extraction.CSharp
         /// <summary>
         /// Gets the source-level modifiers belonging to this symbol, if any.
         /// </summary>
-        public static IEnumerable<string> GetSourceLevelModifiers(this ISymbol symbol)
-        {
-            var methodModifiers = symbol.GetModifiers<Microsoft.CodeAnalysis.CSharp.Syntax.BaseMethodDeclarationSyntax>(md => md.Modifiers);
-            var typeModifiers = symbol.GetModifiers<Microsoft.CodeAnalysis.CSharp.Syntax.TypeDeclarationSyntax>(cd => cd.Modifiers);
-            return methodModifiers.Concat(typeModifiers).Select(m => m.Text);
-        }
+        public static IEnumerable<string> GetSourceLevelModifiers(this ISymbol symbol) =>
+            symbol.GetModifiers<Microsoft.CodeAnalysis.CSharp.Syntax.MemberDeclarationSyntax>(md => md.Modifiers).Select(m => m.Text);
 
         /// <summary>
         /// Holds if the ID generated for `dependant` will contain a reference to
@@ -286,54 +282,60 @@ namespace Semmle.Extraction.CSharp
         public static IEnumerable<IFieldSymbol?> GetTupleElementsMaybeNull(this INamedTypeSymbol type) =>
             type.TupleElements;
 
+        private static void BuildQualifierAndName(INamedTypeSymbol named, Context cx, EscapingTextWriter trapFile, ISymbol symbolBeingDefined)
+        {
+            if (named.ContainingType is not null)
+            {
+                named.ContainingType.BuildOrWriteId(cx, trapFile, symbolBeingDefined, constructUnderlyingTupleType: false);
+                trapFile.Write('.');
+            }
+            else if (named.ContainingNamespace is not null)
+            {
+                if (cx.ShouldAddAssemblyTrapPrefix && named.ContainingAssembly is not null)
+                    BuildAssembly(named.ContainingAssembly, trapFile);
+                named.ContainingNamespace.BuildNamespace(cx, trapFile);
+            }
+
+            var name = named.IsFileLocal ? named.MetadataName : named.Name;
+            trapFile.Write(name);
+        }
+
+        private static void BuildTupleId(INamedTypeSymbol named, Context cx, EscapingTextWriter trapFile, ISymbol symbolBeingDefined)
+        {
+            trapFile.Write('(');
+            trapFile.BuildList(",", named.GetTupleElementsMaybeNull(),
+                (i, f) =>
+                {
+                    if (f is null)
+                    {
+                        trapFile.Write($"null({i})");
+                    }
+                    else
+                    {
+                        trapFile.Write((f.CorrespondingTupleField ?? f).Name);
+                        trapFile.Write(":");
+                        f.Type.BuildOrWriteId(cx, trapFile, symbolBeingDefined, constructUnderlyingTupleType: false);
+                    }
+                }
+                );
+            trapFile.Write(")");
+        }
+
         private static void BuildNamedTypeId(this INamedTypeSymbol named, Context cx, EscapingTextWriter trapFile, ISymbol symbolBeingDefined, bool constructUnderlyingTupleType)
         {
             if (!constructUnderlyingTupleType && named.IsTupleType)
             {
-                trapFile.Write('(');
-                trapFile.BuildList(",", named.GetTupleElementsMaybeNull(),
-                    (i, f) =>
-                    {
-                        if (f is null)
-                        {
-                            trapFile.Write($"null({i})");
-                        }
-                        else
-                        {
-                            trapFile.Write((f.CorrespondingTupleField ?? f).Name);
-                            trapFile.Write(":");
-                            f.Type.BuildOrWriteId(cx, trapFile, symbolBeingDefined, constructUnderlyingTupleType: false);
-                        }
-                    }
-                    );
-                trapFile.Write(")");
+                BuildTupleId(named, cx, trapFile, symbolBeingDefined);
                 return;
             }
 
-            void AddContaining()
-            {
-                if (named.ContainingType is not null)
-                {
-                    named.ContainingType.BuildOrWriteId(cx, trapFile, symbolBeingDefined, constructUnderlyingTupleType: false);
-                    trapFile.Write('.');
-                }
-                else if (named.ContainingNamespace is not null)
-                {
-                    if (cx.ShouldAddAssemblyTrapPrefix && named.ContainingAssembly is not null)
-                        BuildAssembly(named.ContainingAssembly, trapFile);
-                    named.ContainingNamespace.BuildNamespace(cx, trapFile);
-                }
-            }
-
             if (named.TypeParameters.IsEmpty)
             {
-                AddContaining();
-                trapFile.Write(named.Name);
+                BuildQualifierAndName(named, cx, trapFile, symbolBeingDefined);
             }
             else if (named.IsReallyUnbound())
             {
-                AddContaining();
-                trapFile.Write(named.Name);
+                BuildQualifierAndName(named, cx, trapFile, symbolBeingDefined);
                 trapFile.Write("`");
                 trapFile.Write(named.TypeParameters.Length);
             }

csharp/extractor/Semmle.Util/Language.cs

@@ -1,4 +1,4 @@
-namespace Semmle.Autobuild.Shared
+namespace Semmle.Util
 {
     public sealed class Language
     {

csharp/extractor/Semmle.Util/Semmle.Util.csproj

@@ -15,6 +15,7 @@
 
   <ItemGroup>
     <PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
   </ItemGroup>
 
 </Project>

csharp/extractor/Semmle.Util/ToolStatusPage.cs

@@ -0,0 +1,242 @@
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+using Newtonsoft.Json.Serialization;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// Represents diagnostic messages for the tool status page.
+    /// </summary>
+    public class DiagnosticMessage
+    {
+        /// <summary>
+        /// Represents sources of diagnostic messages.
+        /// </summary>
+        public class TspSource
+        {
+            /// <summary>
+            /// An identifier under which it makes sense to group this diagnostic message.
+            /// This is used to build the SARIF reporting descriptor object.
+            /// </summary>
+            public string Id { get; }
+            /// <summary>
+            /// Display name for the ID. This is used to build the SARIF reporting descriptor object.
+            /// </summary>
+            public string Name { get; }
+            /// <summary>
+            /// Name of the CodeQL extractor. This is used to identify which tool component the reporting descriptor object should be nested under in SARIF.
+            /// </summary>
+            public string? ExtractorName { get; }
+
+            public TspSource(string id, string name, string? extractorName = null)
+            {
+                Id = id;
+                Name = name;
+                ExtractorName = extractorName;
+            }
+        }
+
+        /// <summary>
+        /// Enumerates severity levels for diagnostics.
+        /// </summary>
+        [JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]
+        public enum TspSeverity
+        {
+            Note,
+            Warning,
+            Error
+        }
+
+        /// <summary>
+        /// Stores flags indicating where the diagnostic should be displayed.
+        /// </summary>
+        public class TspVisibility
+        {
+            /// <summary>
+            /// A read-only instance of <see cref="TspVisibility" /> which indicates that the
+            /// diagnostic should be used in all supported locations.
+            /// </summary>
+            public static readonly TspVisibility All = new(true, true, true);
+
+            /// <summary>
+            /// True if the message should be displayed on the status page (defaults to false).
+            /// </summary>
+            public bool? StatusPage { get; }
+            /// <summary>
+            /// True if the message should be counted in the diagnostics summary table printed by
+            /// <c>codeql database analyze</c> (defaults to false).
+            /// </summary>
+            public bool? CLISummaryTable { get; }
+            /// <summary>
+            /// True if the message should be sent to telemetry (defaults to false).
+            /// </summary>
+            public bool? Telemetry { get; }
+
+            public TspVisibility(bool? statusPage = null, bool? cliSummaryTable = null, bool? telemetry = null)
+            {
+                this.StatusPage = statusPage;
+                this.CLISummaryTable = cliSummaryTable;
+                this.Telemetry = telemetry;
+            }
+        }
+
+        /// <summary>
+        /// Represents source code locations for diagnostic messages.
+        /// </summary>
+        public class TspLocation
+        {
+            /// <summary>
+            /// Path to the affected file if appropriate, relative to the source root.
+            /// </summary>
+            public string? File { get; }
+            /// <summary>
+            /// The line where the range to which the diagnostic relates to starts.
+            /// </summary>
+            public int? StartLine { get; }
+            /// <summary>
+            /// The column where the range to which the diagnostic relates to starts.
+            /// </summary>
+            public int? StartColumn { get; }
+            /// <summary>
+            /// The line where the range to which the diagnostic relates to ends.
+            /// </summary>
+            public int? EndLine { get; }
+            /// <summary>
+            /// The column where the range to which the diagnostic relates to ends.
+            /// </summary>
+            public int? EndColumn { get; }
+
+            public TspLocation(string? file = null, int? startLine = null, int? startColumn = null, int? endLine = null, int? endColumn = null)
+            {
+                this.File = file;
+                this.StartLine = startLine;
+                this.StartColumn = startColumn;
+                this.EndLine = endLine;
+                this.EndColumn = endColumn;
+            }
+        }
+
+        /// <summary>
+        /// ISO 8601 timestamp.
+        /// </summary>
+        public string Timestamp { get; }
+        /// <summary>
+        /// The source of the diagnostic message.
+        /// </summary>
+        public TspSource Source { get; }
+        /// <summary>
+        /// GitHub flavored Markdown formatted message. Should include inline links to any help pages.
+        /// </summary>
+        public string? MarkdownMessage { get; }
+        /// <summary>
+        /// Plain text message. Used by components where the string processing needed to support
+        /// Markdown is cumbersome.
+        /// </summary>
+        public string? PlaintextMessage { get; }
+        /// <summary>
+        /// List of help links intended to supplement <see cref="PlaintextMessage" />.
+        /// </summary>
+        public List<string> HelpLinks { get; }
+        /// <summary>
+        /// SARIF severity.
+        /// </summary>
+        public TspSeverity? Severity { get; }
+        /// <summary>
+        /// If true, then this message won't be presented to users.
+        /// </summary>
+        public bool Internal { get; }
+        public TspVisibility Visibility { get; }
+        public TspLocation Location { get; }
+        /// <summary>
+        /// Structured metadata about the diagnostic message.
+        /// </summary>
+        public Dictionary<string, object> Attributes { get; }
+
+        public DiagnosticMessage(
+            Language language, string id, string name, string? markdownMessage = null, string? plaintextMessage = null,
+            TspVisibility? visibility = null, TspLocation? location = null, TspSeverity? severity = TspSeverity.Error,
+            DateTime? timestamp = null, bool? intrnl = null
+        )
+        {
+            this.Source = new TspSource(
+                id: $"{language.UpperCaseName.ToLower()}/autobuilder/{id}",
+                name: name,
+                extractorName: language.UpperCaseName.ToLower()
+            );
+            this.Timestamp = (timestamp ?? DateTime.UtcNow).ToString("o", CultureInfo.InvariantCulture);
+            this.HelpLinks = new List<string>();
+            this.Attributes = new Dictionary<string, object>();
+            this.Severity = severity;
+            this.Visibility = visibility ?? TspVisibility.All;
+            this.Location = location ?? new TspLocation();
+            this.Internal = intrnl ?? false;
+            this.MarkdownMessage = markdownMessage;
+            this.PlaintextMessage = plaintextMessage;
+        }
+    }
+
+    /// <summary>
+    /// Provides the ability to write diagnostic messages to some output.
+    /// </summary>
+    public interface IDiagnosticsWriter
+    {
+        /// <summary>
+        /// Adds <paramref name="message" /> as a new diagnostics entry.
+        /// </summary>
+        /// <param name="message">The diagnostics entry to add.</param>
+        void AddEntry(DiagnosticMessage message);
+    }
+
+    /// <summary>
+    /// A wrapper around an underlying <see cref="StreamWriter" /> which allows
+    /// <see cref="DiagnosticMessage" /> objects to be serialized to it.
+    /// </summary>
+    public sealed class DiagnosticsStream : IDiagnosticsWriter, IDisposable
+    {
+        private readonly JsonSerializer serializer;
+        private readonly StreamWriter writer;
+
+        /// <summary>
+        /// Initialises a new <see cref="DiagnosticsStream" /> for a file at <paramref name="path" />.
+        /// </summary>
+        /// <param name="path">The path to the file that should be created.</param>
+        public DiagnosticsStream(string path)
+        {
+            this.writer = File.CreateText(path);
+
+            var contractResolver = new DefaultContractResolver
+            {
+                NamingStrategy = new CamelCaseNamingStrategy()
+            };
+
+            serializer = new JsonSerializer
+            {
+                ContractResolver = contractResolver,
+                NullValueHandling = NullValueHandling.Ignore
+            };
+        }
+
+        /// <summary>
+        /// Adds <paramref name="message" /> as a new diagnostics entry.
+        /// </summary>
+        /// <param name="message">The diagnostics entry to add.</param>
+        public void AddEntry(DiagnosticMessage message)
+        {
+            serializer.Serialize(writer, message);
+            writer.Flush();
+        }
+
+        /// <summary>
+        /// Releases all resources used by the <see cref="DiagnosticsStream" /> object.
+        /// </summary>
+        public void Dispose()
+        {
+            writer.Dispose();
+        }
+    }
+}

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.4.3
+version: 1.4.4-dev
 groups:
     - csharp
     - solorigate

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.4.3
+version: 1.4.4-dev
 groups:
     - csharp
     - solorigate