C#: Use dependency injection in the auto builder for Diagnostic classifier.

Refactor `GetDiagnosticSource` into `MakeDiagnostic`
which sets the defaults.

.github/actions/find-latest-bundle/action.yml

@@ -1,26 +0,0 @@
-name: Find Latest CodeQL Bundle
-description: Finds the URL of the latest released version of the CodeQL bundle.
-outputs:
-  url:
-    description: The download URL of the latest CodeQL bundle release
-    value: ${{ steps.find-latest.outputs.url }}
-runs:
-  using: composite
-  steps:
-    - name: Find Latest Release
-      id: find-latest
-      shell: pwsh
-      run: |
-        $Latest = gh release list --repo github/codeql-action --exclude-drafts --limit 1000 |
-          ForEach-Object { $C = $_ -split "`t"; return @{ type = $C[1]; tag = $C[2]; } } |
-          Where-Object { $_.type -eq 'Latest' }
-
-        $Tag = $Latest.tag
-        if ($Tag -eq '') {
-          throw 'Failed to find latest bundle release.'
-        }
-
-        Write-Output "Latest bundle tag is '${Tag}'."
-        "url=https://github.com/github/codeql-action/releases/download/${Tag}/codeql-bundle-linux64.tar.gz" >> $env:GITHUB_OUTPUT
-      env:
-        GITHUB_TOKEN: ${{ github.token }}

.github/workflows/codeql-analysis.yml

@@ -28,7 +28,7 @@ jobs:
 
     steps:
     - name: Setup dotnet
-      uses: actions/setup-dotnet@v2
+      uses: actions/setup-dotnet@v3
       with:
         dotnet-version: 7.0.102
 

.github/workflows/go-tests-other-os.yml

@@ -12,10 +12,10 @@ jobs:
     name: Test MacOS
     runs-on: macos-latest
     steps:
-      - name: Set up Go 1.19
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.20.0
         id: go
 
       - name: Check out code
@@ -47,10 +47,10 @@ jobs:
     name: Test Windows
     runs-on: windows-latest-xl
     steps:
-      - name: Set up Go 1.19
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.20.0
         id: go
 
       - name: Check out code

.github/workflows/go-tests.yml

@@ -20,10 +20,10 @@ jobs:
     name: Test Linux (Ubuntu)
     runs-on: ubuntu-latest-xl
     steps:
-      - name: Set up Go 1.19
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.20.0
         id: go
 
       - name: Check out code

.github/workflows/ql-for-ql-build.yml

@@ -22,66 +22,22 @@ jobs:
     steps:
       ### Build the queries ###
       - uses: actions/checkout@v3
-      - name: Find latest bundle
-        id: find-latest-bundle
-        uses: ./.github/actions/find-latest-bundle
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
+        uses: github/codeql-action/init@v2
         with:
           languages: javascript # does not matter
-          tools: ${{ steps.find-latest-bundle.outputs.url }}
-      - name: Get CodeQL version
-        id: get-codeql-version
-        run: |
-          echo "version=$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)" >> $GITHUB_OUTPUT
-        shell: bash
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - uses: ./.github/actions/os-version
         id: os_version
-      - name: Cache entire pack
-        id: cache-pack
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/pack
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-pack-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
-      - name: Cache queries
-        if: steps.cache-pack.outputs.cache-hit != 'true'
-        id: cache-queries
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/queries
-          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
-      - name: Build query pack
-        if: steps.cache-queries.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: |
-          cd ql/ql/src
-          "${CODEQL}" pack create -j 16
-          mv .codeql/pack/codeql/ql/0.0.0 ${{ runner.temp }}/queries
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Move cache queries to pack
-        if: steps.cache-pack.outputs.cache-hit != 'true'
-        run: |
-          cp -r ${{ runner.temp }}/queries ${{ runner.temp }}/pack
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-
       ### Build the extractor ###
       - name: Cache entire extractor
-        if: steps.cache-pack.outputs.cache-hit != 'true'
         id: cache-extractor
         uses: actions/cache@v3
         with:
-          path: |
-            ql/target/release/ql-autobuilder
-            ql/target/release/ql-autobuilder.exe
-            ql/target/release/ql-extractor
-            ql/target/release/ql-extractor.exe
+          path: ql/extractor-pack/
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
       - name: Cache cargo
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         uses: actions/cache@v3
         with:
           path: |
@@ -89,77 +45,31 @@ jobs:
             ~/.cargo/git
             ql/target
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
-      - name: Check formatting
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: cd ql; cargo fmt --all -- --check
-      - name: Build
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: cd ql; cargo build --verbose
-      - name: Run tests
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: cd ql; cargo test --verbose
       - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: cd ql; cargo build --release
-      - name: Generate dbscheme
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
-
-      ### Package the queries and extractor ###
-      - name: Package pack
-        if: steps.cache-pack.outputs.cache-hit != 'true'
-        run: |
-          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats ${PACK}/
-          mkdir -p ${PACK}/tools/linux64
-          cp ql/target/release/ql-autobuilder  ${PACK}/tools/linux64/autobuilder
-          cp ql/target/release/ql-extractor ${PACK}/tools/linux64/extractor
-          chmod +x ${PACK}/tools/linux64/autobuilder
-          chmod +x ${PACK}/tools/linux64/extractor
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        run: cd ql; ./scripts/create-extractor-pack.sh       
         env:
-          PACK: ${{ runner.temp }}/pack
-
-      ### Run the analysis ###
-      - name: Hack codeql-action options
+          GH_TOKEN: ${{ github.token }}   
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: run-ql-for-ql
+      - name: Make database and analyze
         run: |
-          JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .resolve.languages=["--search-path", $pack] | .database.init=["--search-path", $pack]')
-          echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
-        env:
-          PACK: ${{ runner.temp }}/pack
-
-      - name: Create CodeQL config file
-        run: |
-          echo "paths-ignore:" >> ${CONF}
-          echo "  - ql/ql/test" >> ${CONF}
-          echo "  - \"*/ql/lib/upgrades/\"" >> ${CONF}
-          echo "disable-default-queries: true" >> ${CONF}
-          echo "queries:" >> ${CONF}
-          echo "  - uses: ./ql/ql/src/codeql-suites/ql-code-scanning.qls" >> ${CONF}
-          echo "Config file: "
-          cat ${CONF}
-        env:
-          CONF: ./ql-for-ql-config.yml
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
+          ${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB}
+          ${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls  --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        env: 
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+          DB: ${{ runner.temp }}/DB
+          LGTM_INDEX_FILTERS: |
+            exclude:ql/ql/test
+            exclude:*/ql/lib/upgrades/
+      - name: Upload sarif to code-scanning
+        uses: github/codeql-action/upload-sarif@v2
         with:
-          languages: ql
-          db-location: ${{ runner.temp }}/db
-          config-file: ./ql-for-ql-config.yml
-          tools: ${{ steps.find-latest-bundle.outputs.url }}
-      - name: Move pack queries
-        run: |
-          cp -r ${PACK}/queries ql/ql/src
-        env:
-          PACK: ${{ runner.temp }}/pack
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
-        with:
-          category: "ql-for-ql"
-      - name: Copy sarif file to CWD
-        run: cp ../results/ql.sarif ./ql-for-ql.sarif
-      - name: Fixup the $scema in sarif  # Until https://github.com/microsoft/sarif-vscode-extension/pull/436/ is part in a stable release
-        run: |
-          sed -i 's/\$schema.*/\$schema": "https:\/\/raw.githubusercontent.com\/oasis-tcs\/sarif-spec\/master\/Schemata\/sarif-schema-2.1.0",/' ql-for-ql.sarif
+          sarif_file: ql-for-ql.sarif
+          category: ql-for-ql
       - name: Sarif as artifact
         uses: actions/upload-artifact@v3
         with:

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -25,7 +25,7 @@ jobs:
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
+        uses: github/codeql-action/init@v2
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version

.github/workflows/ql-for-ql-tests.yml

@@ -24,7 +24,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
+        uses: github/codeql-action/init@v2
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version
@@ -36,6 +36,8 @@ jobs:
             ~/.cargo/git
             ql/target
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/rust-toolchain.toml', 'ql/**/Cargo.lock') }}
+      - name: Check formatting
+        run: cd ql; cargo fmt --all -- --check
       - name: Build extractor
         run: |
           cd ql;
@@ -67,7 +69,7 @@ jobs:
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
+        uses: github/codeql-action/init@v2
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -1,5 +1,6 @@
 using Xunit;
 using Semmle.Autobuild.Shared;
+using Semmle.Util;
 using System.Collections.Generic;
 using System;
 using System.Linq;
@@ -75,6 +76,15 @@ namespace Semmle.Autobuild.Cpp.Tests
             throw new ArgumentException("Missing RunProcess " + pattern);
         }
 
+        int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
+        {
+            var ret = (this as IBuildActions).RunProcess(cmd, args, workingDirectory, env, out var stdout);
+
+            stdout.ForEach(line => onOutput(line));
+
+            return ret;
+        }
+
         public IList<string> DirectoryDeleteIn = new List<string>();
 
         void IBuildActions.DirectoryDelete(string dir, bool recursive)
@@ -131,6 +141,14 @@ namespace Semmle.Autobuild.Cpp.Tests
 
         bool IBuildActions.IsWindows() => IsWindows;
 
+        public bool IsMacOs { get; set; }
+
+        bool IBuildActions.IsMacOs() => IsMacOs;
+
+        public bool IsArm { get; set; }
+
+        bool IBuildActions.IsArm() => IsArm;
+
         string IBuildActions.PathCombine(params string[] parts)
         {
             return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));
@@ -235,6 +253,7 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
+            Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = Path.GetTempPath();
             Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
             Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";

cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs

@@ -1,4 +1,5 @@
 using Semmle.Autobuild.Shared;
+using Semmle.Util;
 
 namespace Semmle.Autobuild.Cpp
 {
@@ -21,7 +22,7 @@ namespace Semmle.Autobuild.Cpp
 
     public class CppAutobuilder : Autobuilder<CppAutobuildOptions>
     {
-        public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options) { }
+        public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options, new DiagnosticClassifier()) { }
 
         public override BuildScript GetBuildScript()
         {

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll

@@ -707,8 +707,8 @@ private module Cached {
      * Gets a viable dispatch target of `call` in the context `ctx`. This is
      * restricted to those `call`s for which a context might make a difference.
      */
-    pragma[nomagic]
-    private DataFlowCallable viableImplInCallContextExt(DataFlowCall call, DataFlowCall ctx) {
+    cached
+    DataFlowCallable viableImplInCallContextExt(DataFlowCall call, DataFlowCall ctx) {
       result = viableImplInCallContext(call, ctx) and
       result = viableCallable(call)
       or
@@ -1391,6 +1391,9 @@ class TypedContentApprox extends MkTypedContentApprox {
   /** Gets a typed content approximated by this value. */
   TypedContent getATypedContent() { result = getATypedContent(this) }
 
+  /** Gets the content. */
+  ContentApprox getContent() { result = c }
+
   /** Gets the container type. */
   DataFlowType getContainerType() { result = t }
 
@@ -1408,6 +1411,8 @@ abstract class ApproxAccessPathFront extends TApproxAccessPathFront {
 
   abstract boolean toBoolNonEmpty();
 
+  TypedContentApprox getHead() { this = TApproxFrontHead(result) }
+
   pragma[nomagic]
   TypedContent getAHead() {
     exists(TypedContentApprox cont |

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.5.2
+version: 0.5.3-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl4.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll

@@ -707,8 +707,8 @@ private module Cached {
      * Gets a viable dispatch target of `call` in the context `ctx`. This is
      * restricted to those `call`s for which a context might make a difference.
      */
-    pragma[nomagic]
-    private DataFlowCallable viableImplInCallContextExt(DataFlowCall call, DataFlowCall ctx) {
+    cached
+    DataFlowCallable viableImplInCallContextExt(DataFlowCall call, DataFlowCall ctx) {
       result = viableImplInCallContext(call, ctx) and
       result = viableCallable(call)
       or
@@ -1391,6 +1391,9 @@ class TypedContentApprox extends MkTypedContentApprox {
   /** Gets a typed content approximated by this value. */
   TypedContent getATypedContent() { result = getATypedContent(this) }
 
+  /** Gets the content. */
+  ContentApprox getContent() { result = c }
+
   /** Gets the container type. */
   DataFlowType getContainerType() { result = t }
 
@@ -1408,6 +1411,8 @@ abstract class ApproxAccessPathFront extends TApproxAccessPathFront {
 
   abstract boolean toBoolNonEmpty();
 
+  TypedContentApprox getHead() { this = TApproxFrontHead(result) }
+
   pragma[nomagic]
   TypedContent getAHead() {
     exists(TypedContentApprox cont |

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplLocal.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll

@@ -707,8 +707,8 @@ private module Cached {
      * Gets a viable dispatch target of `call` in the context `ctx`. This is
      * restricted to those `call`s for which a context might make a difference.
      */
-    pragma[nomagic]
-    private DataFlowCallable viableImplInCallContextExt(DataFlowCall call, DataFlowCall ctx) {
+    cached
+    DataFlowCallable viableImplInCallContextExt(DataFlowCall call, DataFlowCall ctx) {
       result = viableImplInCallContext(call, ctx) and
       result = viableCallable(call)
       or
@@ -1391,6 +1391,9 @@ class TypedContentApprox extends MkTypedContentApprox {
   /** Gets a typed content approximated by this value. */
   TypedContent getATypedContent() { result = getATypedContent(this) }
 
+  /** Gets the content. */
+  ContentApprox getContent() { result = c }
+
   /** Gets the container type. */
   DataFlowType getContainerType() { result = t }
 
@@ -1408,6 +1411,8 @@ abstract class ApproxAccessPathFront extends TApproxAccessPathFront {
 
   abstract boolean toBoolNonEmpty();
 
+  TypedContentApprox getHead() { this = TApproxFrontHead(result) }
+
   pragma[nomagic]
   TypedContent getAHead() {
     exists(TypedContentApprox cont |

cpp/ql/lib/semmle/code/cpp/security/Encryption.qll

@@ -64,7 +64,7 @@ predicate isInsecureEncryption(string name) { name.regexpMatch(getInsecureAlgori
  */
 bindingset[name]
 predicate isEncryptionAdditionalEvidence(string name) {
-  name.toUpperCase().matches("%" + ["CRYPT", "CODE", "CODING", "CBC", "KEY", "CIPHER", "MAC"] + "%")
+  name.regexpMatch("(?i).*(crypt|code|coding|cbc|key|cipher|mac).*")
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/security/SensitiveExprs.qll

@@ -14,8 +14,8 @@ import cpp
  */
 bindingset[s]
 private predicate suspicious(string s) {
-  s.regexpMatch(".*(password|passwd|accountid|account.?key|accnt.?key|license.?key|trusted).*") and
-  not s.matches(["%hash%", "%crypt%", "%file%", "%path%", "%invalid%"])
+  s.regexpMatch("(?i).*(password|passwd|accountid|account.?key|accnt.?key|license.?key|trusted).*") and
+  not s.regexpMatch("(?i).*(hash|crypt|file|path|invalid).*")
 }
 
 /**
@@ -23,7 +23,7 @@ private predicate suspicious(string s) {
  */
 class SensitiveVariable extends Variable {
   SensitiveVariable() {
-    suspicious(this.getName().toLowerCase()) and
+    suspicious(this.getName()) and
     not this.getUnspecifiedType() instanceof IntegralType
   }
 }
@@ -33,7 +33,7 @@ class SensitiveVariable extends Variable {
  */
 class SensitiveFunction extends Function {
   SensitiveFunction() {
-    suspicious(this.getName().toLowerCase()) and
+    suspicious(this.getName()) and
     not this.getUnspecifiedType() instanceof IntegralType
   }
 }

cpp/ql/src/experimental/Security/CWE/CWE-805/BufferAccessWithIncorrectLengthValue.cpp

@@ -0,0 +1,8 @@
+
+...
+char buf[256];
+X509_NAME_oneline(X509_get_subject_name(peer),buf,sizeof(buf)); // GOOD
+...
+char buf[256];
+X509_NAME_oneline(X509_get_subject_name(peer),buf,1024); // BAD
+...

cpp/ql/src/experimental/Security/CWE/CWE-805/BufferAccessWithIncorrectLengthValue.qhelp

@@ -0,0 +1,23 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+<overview>
+<p>Using a size argument that is larger than the buffer size will result in an out-of-bounds memory access and possibly overflow. You need to limit the value of the length argument.</p>
+
+</overview>
+
+<example>
+<p>The following example shows the use of a function with and without an error in the size argument.</p>
+<sample src="BufferAccessWithIncorrectLengthValue.cpp" />
+
+</example>
+<references>
+
+<li>
+  CERT Coding Standard:
+  <a href="https://wiki.sei.cmu.edu/confluence/display/c/ARR38-C.+Guarantee+that+library+functions+do+not+form+invalid+pointers">ARR38-C. Guarantee that library functions do not form invalid pointers - SEI CERT C Coding Standard - Confluence</a>.
+</li>
+
+</references>
+</qhelp>

cpp/ql/src/experimental/Security/CWE/CWE-805/BufferAccessWithIncorrectLengthValue.ql

@@ -0,0 +1,76 @@
+/**
+ * @name Buffer access with incorrect length value
+ * @description Incorrect use of the length argument in some functions will result in out-of-memory accesses.
+ * @kind problem
+ * @id cpp/buffer-access-with-incorrect-length-value
+ * @problem.severity warning
+ * @precision medium
+ * @tags correctness
+ *       security
+ *       experimental
+ *       external/cwe/cwe-805
+ */
+
+import cpp
+
+/** Holds for a function `f`, which has an argument at index `bpos` that points to a buffer and an argument at index `spos` that points to a size. */
+predicate numberArgument(Function f, int bpos, int spos) {
+  f.hasGlobalOrStdName([
+      "X509_NAME_oneline", "SSL_CIPHER_description", "SSL_get_shared_ciphers",
+      "SSL_export_keying_material_early", "SSL_export_keying_material", "SSL_set_alpn_protos",
+      "SSL_CTX_set_alpn_protos", "SSL_read", "SSL_read_ex", "SSL_read_early_data",
+      "SSL_bytes_to_cipher_list", "SSL_write", "SSL_SESSION_set1_master_key",
+      "SSL_CTX_set_session_id_context", "BIO_gets", "BIO_read", "BIO_read_ex", "BIO_write",
+      "BIO_write_ex", "BIO_ctrl", "BN_bn2binpad", "BN_signed_bn2bin", "BN_signed_bn2lebin",
+      "EVP_PKEY_get_default_digest_name", "EVP_DigestUpdate", "EVP_PKEY_CTX_set1_tls1_prf_secret",
+      "EVP_KDF_derive", "EVP_CIPHER_CTX_get_updated_iv", "EVP_PKEY_get_group_name", "EVP_MAC_init",
+      "write", "read", "send", "sendto", "recv", "recvfrom", "strerror_r"
+    ]) and
+  bpos = 1 and
+  spos = 2
+  or
+  f.hasGlobalOrStdName(["X509_NAME_get_text_by_NID", "EVP_PKEY_get_utf8_string_param"]) and
+  bpos = 2 and
+  spos = 3
+  or
+  f.hasGlobalOrStdName([
+      "BIO_snprintf", "BN_signed_lebin2bn", "BIO_new_mem_buf", "BN_lebin2bn", "BN_bin2bn",
+      "EVP_read_pw_string", "EVP_read_pw_string", "strftime", "strnlen", "fgets", "snprintf",
+      "vsnprintf"
+    ]) and
+  bpos = 0 and
+  spos = 1
+  or
+  f.hasGlobalOrStdName(["AES_ige_encrypt", "memchr"]) and bpos = 0 and spos = 2
+  or
+  f.hasGlobalOrStdName("EVP_MAC_final") and bpos = 1 and spos = 3
+  or
+  f.hasGlobalOrStdName("OBJ_obj2txt") and bpos = 2 and spos = 1
+  or
+  f.hasGlobalOrStdName("EVP_CIPHER_CTX_ctrl") and bpos = 3 and spos = 2
+  or
+  f.hasGlobalOrStdName(["EVP_PKEY_get_octet_string_param", "getnameinfo"]) and bpos = 2 and spos = 3
+  or
+  f.hasGlobalOrStdName([
+      "EVP_DecryptUpdate", "EVP_EncryptUpdate", "EVP_PKEY_encrypt", "EVP_PKEY_sign",
+      "EVP_CipherUpdate"
+    ]) and
+  bpos = 3 and
+  spos = 4
+  or
+  f.hasGlobalOrStdName("getnameinfo") and bpos = 4 and spos = 5
+}
+
+from FunctionCall fc
+where
+  exists(ArrayType array, int bufArgPos, int sizeArgPos |
+    numberArgument(fc.getTarget(), bufArgPos, sizeArgPos) and
+    fc.getArgument(pragma[only_bind_into](sizeArgPos)).getValue().toInt() > array.getByteSize() and
+    fc.getArgument(pragma[only_bind_into](bufArgPos))
+        .(VariableAccess)
+        .getTarget()
+        .getADeclarationEntry()
+        .getType() = array
+  )
+select fc,
+  "Access beyond the bounds of the allocated memory is possible, the size argument used is greater than the size of the buffer."

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.5.2
+version: 0.5.3-dev
 groups: 
   - cpp
   - queries

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-805/semmle/tests/BufferAccessWithIncorrectLengthValue.expected

@@ -0,0 +1 @@
+| test.cpp:27:5:27:21 | call to X509_NAME_oneline | Access beyond the bounds of the allocated memory is possible, the size argument used is greater than the size of the buffer. |

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-805/semmle/tests/BufferAccessWithIncorrectLengthValue.qlref

@@ -0,0 +1 @@
+experimental/Security/CWE/CWE-805/BufferAccessWithIncorrectLengthValue.ql

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-805/semmle/tests/test.cpp

@@ -0,0 +1,31 @@
+struct X509_NAME {};
+struct SSL {};
+struct X509 {};
+
+char * X509_NAME_oneline(X509_NAME *a,char *buf,int size);
+X509 *SSL_get_peer_certificate(const SSL *ssl);
+X509_NAME *X509_get_subject_name(const X509 *x);
+char *strcasestr(char *a, char *b);
+
+bool goodTest1(SSL *ssl,char *text)
+{
+  X509 *peer;
+  char buf[256];
+  if( peer = SSL_get_peer_certificate(ssl))
+  {
+    X509_NAME_oneline(X509_get_subject_name(peer),buf,sizeof(buf)); // GOOD
+    if((char*)strcasestr(buf,text)) return true;
+  }
+  return false;
+}
+bool badTest1(SSL *ssl,char *text)
+{
+  X509 *peer;
+  char buf[256];
+  if( peer = SSL_get_peer_certificate(ssl))
+  {
+    X509_NAME_oneline(X509_get_subject_name(peer),buf,1024); // BAD
+    if((char*)strcasestr(buf,text)) return true;
+  }
+  return false;
+}

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -1,5 +1,6 @@
 using Xunit;
 using Semmle.Autobuild.Shared;
+using Semmle.Util;
 using System.Collections.Generic;
 using System;
 using System.Linq;
@@ -85,6 +86,15 @@ namespace Semmle.Autobuild.CSharp.Tests
             return ret;
         }
 
+        int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
+        {
+            var ret = (this as IBuildActions).RunProcess(cmd, args, workingDirectory, env, out var stdout);
+
+            stdout.ForEach(line => onOutput(line));
+
+            return ret;
+        }
+
         public IList<string> DirectoryDeleteIn { get; } = new List<string>();
 
         void IBuildActions.DirectoryDelete(string dir, bool recursive)
@@ -145,6 +155,14 @@ namespace Semmle.Autobuild.CSharp.Tests
 
         bool IBuildActions.IsWindows() => IsWindows;
 
+        public bool IsMacOs { get; set; }
+
+        bool IBuildActions.IsMacOs() => IsMacOs;
+
+        public bool IsArm { get; set; }
+
+        bool IBuildActions.IsArm() => IsArm;
+
         public string PathCombine(params string[] parts)
         {
             return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));
@@ -383,6 +401,7 @@ namespace Semmle.Autobuild.CSharp.Tests
             actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
             actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
+            actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = Path.GetTempPath();
             actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = isWindows ? "win64" : "linux64";
             actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;

csharp/autobuilder/Semmle.Autobuild.CSharp/AutoBuildRule.cs

@@ -0,0 +1,69 @@
+using Semmle.Autobuild.Shared;
+using Semmle.Extraction.CSharp;
+
+namespace Semmle.Autobuild.CSharp
+{
+    internal class AutoBuildRule : IBuildRule<CSharpAutobuildOptions>
+    {
+        private readonly CSharpAutobuilder autobuilder;
+
+        public DotNetRule DotNetRule { get; }
+
+        public MsBuildRule MsBuildRule { get; }
+
+        public BuildCommandAutoRule BuildCommandAutoRule { get; }
+
+        public AutoBuildRule(CSharpAutobuilder autobuilder)
+        {
+            this.autobuilder = autobuilder;
+            this.DotNetRule = new DotNetRule();
+            this.MsBuildRule = new MsBuildRule();
+            this.BuildCommandAutoRule = new BuildCommandAutoRule(DotNetRule.WithDotNet);
+        }
+
+        public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
+        {
+            var cleanTrapFolder =
+                BuildScript.DeleteDirectory(this.autobuilder.TrapDir);
+            var cleanSourceArchive =
+                BuildScript.DeleteDirectory(this.autobuilder.SourceArchiveDir);
+            var tryCleanExtractorArgsLogs =
+                BuildScript.Create(actions =>
+                {
+                    foreach (var file in Extractor.GetCSharpArgsLogs())
+                    {
+                        try
+                        {
+                            actions.FileDelete(file);
+                        }
+                        catch // lgtm[cs/catch-of-all-exceptions] lgtm[cs/empty-catch-block]
+                        { }
+                    }
+
+                    return 0;
+                });
+            var attemptExtractorCleanup =
+                BuildScript.Try(cleanTrapFolder) &
+                BuildScript.Try(cleanSourceArchive) &
+                tryCleanExtractorArgsLogs &
+                BuildScript.DeleteFile(Extractor.GetCSharpLogPath());
+
+            /// <summary>
+            /// Execute script `s` and check that the C# extractor has been executed.
+            /// If either fails, attempt to cleanup any artifacts produced by the extractor,
+            /// and exit with code 1, in order to proceed to the next attempt.
+            /// </summary>
+            BuildScript IntermediateAttempt(BuildScript s) =>
+                (s & this.autobuilder.CheckExtractorRun(false)) |
+                (attemptExtractorCleanup & BuildScript.Failure);
+
+            return
+                // First try .NET Core
+                IntermediateAttempt(this.DotNetRule.Analyse(this.autobuilder, true)) |
+                // Then MSBuild
+                (() => IntermediateAttempt(this.MsBuildRule.Analyse(this.autobuilder, true))) |
+                // And finally look for a script that might be a build script
+                (() => this.BuildCommandAutoRule.Analyse(this.autobuilder, true) & this.autobuilder.CheckExtractorRun(true));
+        }
+    }
+}

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

@@ -1,6 +1,8 @@
 using Semmle.Extraction.CSharp;
 using Semmle.Util.Logging;
 using Semmle.Autobuild.Shared;
+using Semmle.Util;
+using System.Linq;
 
 namespace Semmle.Autobuild.CSharp
 {
@@ -29,25 +31,16 @@ namespace Semmle.Autobuild.CSharp
 
     public class CSharpAutobuilder : Autobuilder<CSharpAutobuildOptions>
     {
-        public CSharpAutobuilder(IBuildActions actions, CSharpAutobuildOptions options) : base(actions, options) { }
+        private const string buildCommandDocsUrl =
+            "https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages";
+
+        private readonly AutoBuildRule autoBuildRule;
+
+        public CSharpAutobuilder(IBuildActions actions, CSharpAutobuildOptions options) : base(actions, options, new CSharpDiagnosticClassifier()) =>
+            this.autoBuildRule = new AutoBuildRule(this);
 
         public override BuildScript GetBuildScript()
         {
-            /// <summary>
-            /// A script that checks that the C# extractor has been executed.
-            /// </summary>
-            BuildScript CheckExtractorRun(bool warnOnFailure) =>
-                BuildScript.Create(actions =>
-                {
-                    if (actions.FileExists(Extractor.GetCSharpLogPath()))
-                        return 0;
-
-                    if (warnOnFailure)
-                        Log(Severity.Error, "No C# code detected during build.");
-
-                    return 1;
-                });
-
             var attempt = BuildScript.Failure;
             switch (GetCSharpBuildStrategy())
             {
@@ -65,47 +58,9 @@ namespace Semmle.Autobuild.CSharp
                     attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true);
                     break;
                 case CSharpBuildStrategy.Auto:
-                    var cleanTrapFolder =
-                        BuildScript.DeleteDirectory(TrapDir);
-                    var cleanSourceArchive =
-                        BuildScript.DeleteDirectory(SourceArchiveDir);
-                    var tryCleanExtractorArgsLogs =
-                        BuildScript.Create(actions =>
-                        {
-                            foreach (var file in Extractor.GetCSharpArgsLogs())
-                            {
-                                try
-                                {
-                                    actions.FileDelete(file);
-                                }
-                                catch // lgtm[cs/catch-of-all-exceptions] lgtm[cs/empty-catch-block]
-                                { }
-                            }
-
-                            return 0;
-                        });
-                    var attemptExtractorCleanup =
-                        BuildScript.Try(cleanTrapFolder) &
-                        BuildScript.Try(cleanSourceArchive) &
-                        tryCleanExtractorArgsLogs &
-                        BuildScript.DeleteFile(Extractor.GetCSharpLogPath());
-
-                    /// <summary>
-                    /// Execute script `s` and check that the C# extractor has been executed.
-                    /// If either fails, attempt to cleanup any artifacts produced by the extractor,
-                    /// and exit with code 1, in order to proceed to the next attempt.
-                    /// </summary>
-                    BuildScript IntermediateAttempt(BuildScript s) =>
-                        (s & CheckExtractorRun(false)) |
-                        (attemptExtractorCleanup & BuildScript.Failure);
-
                     attempt =
-                        // First try .NET Core
-                        IntermediateAttempt(new DotNetRule().Analyse(this, true)) |
-                        // Then MSBuild
-                        (() => IntermediateAttempt(new MsBuildRule().Analyse(this, true))) |
-                        // And finally look for a script that might be a build script
-                        (() => new BuildCommandAutoRule(DotNetRule.WithDotNet).Analyse(this, true) & CheckExtractorRun(true)) |
+                        // Attempt a few different build strategies to see if one works
+                        this.autoBuildRule.Analyse(this, true) |
                         // All attempts failed: print message
                         AutobuildFailure();
                     break;
@@ -114,6 +69,127 @@ namespace Semmle.Autobuild.CSharp
             return attempt;
         }
 
+        /// <summary>
+        /// A script that checks that the C# extractor has been executed.
+        /// </summary>
+        public BuildScript CheckExtractorRun(bool warnOnFailure) =>
+            BuildScript.Create(actions =>
+            {
+                if (actions.FileExists(Extractor.GetCSharpLogPath()))
+                    return 0;
+
+                if (warnOnFailure)
+                    Log(Severity.Error, "No C# code detected during build.");
+
+                return 1;
+            });
+
+        protected override void AutobuildFailureDiagnostic()
+        {
+            // if `ScriptPath` is not null here, the `BuildCommandAuto` rule was
+            // run and found at least one script to execute
+            if (this.autoBuildRule.BuildCommandAutoRule.ScriptPath is not null)
+            {
+                var relScriptPath = this.MakeRelative(autoBuildRule.BuildCommandAutoRule.ScriptPath);
+
+                // if we found multiple build scripts in the project directory, then we can say
+                // as much to indicate that we may have picked the wrong one;
+                // otherwise, we just report that the one script we found didn't work
+                DiagnosticMessage message =
+                    this.autoBuildRule.BuildCommandAutoRule.CandidatePaths.Count() > 1 ?
+                    new(
+                        this.Options.Language,
+                        "multiple-build-scripts",
+                        "There are multiple potential build scripts",
+                        markdownMessage:
+                            "CodeQL found multiple potential build scripts for your project and " +
+                            $"attempted to run `{relScriptPath}`, which failed. " +
+                            "This may not be the right build script for your project. " +
+                            $"Set up a [manual build command]({buildCommandDocsUrl})."
+                    ) :
+                    new(
+                        this.Options.Language,
+                        "script-failure",
+                        "Unable to build project using build script",
+                        markdownMessage:
+                            "CodeQL attempted to build your project using a script located at " +
+                            $"`{relScriptPath}`, which failed. " +
+                            $"Set up a [manual build command]({buildCommandDocsUrl})."
+                    );
+
+                AddDiagnostic(message);
+            }
+
+            // project files which don't exist get marked as not .NET core projects, but we don't want
+            // to show an error for this if the files don't exist
+            var foundNotDotNetProjects = autoBuildRule.DotNetRule.NotDotNetProjects.Where(
+                proj => this.Actions.FileExists(proj.FullPath)
+            );
+
+            // both dotnet and msbuild builds require project or solution files; if we haven't found any
+            // then neither of those rules would've worked
+            if (this.ProjectsOrSolutionsToBuild.Count == 0)
+            {
+                this.AddDiagnostic(new(
+                    this.Options.Language,
+                    "no-projects-or-solutions",
+                    "No project or solutions files found",
+                    markdownMessage:
+                        "CodeQL could not find any project or solution files in your repository. " +
+                        $"Set up a [manual build command]({buildCommandDocsUrl})."
+                ));
+            }
+            // show a warning if there are projects which are not compatible with .NET Core, in case that is unintentional
+            else if (foundNotDotNetProjects.Any())
+            {
+                this.AddDiagnostic(new(
+                    this.Options.Language,
+                    "dotnet-incompatible-projects",
+                    "Some projects are incompatible with .NET Core",
+                    severity: DiagnosticMessage.TspSeverity.Warning,
+                    markdownMessage: $"""
+                    CodeQL found some projects which cannot be built with .NET Core:
+
+                    {autoBuildRule.DotNetRule.NotDotNetProjects.Select(p => this.MakeRelative(p.FullPath)).ToMarkdownList(MarkdownUtil.CodeFormatter, 5)}
+                    """
+                ));
+            }
+
+            // report any projects that failed to build with .NET Core
+            if (autoBuildRule.DotNetRule.FailedProjectsOrSolutions.Any())
+            {
+                this.AddDiagnostic(new(
+                    this.Options.Language,
+                    "dotnet-build-failure",
+                    "Some projects or solutions failed to build using .NET Core",
+                    markdownMessage: $"""
+                    CodeQL was unable to build the following projects using .NET Core:
+
+                    {autoBuildRule.DotNetRule.FailedProjectsOrSolutions.Select(p => this.MakeRelative(p.FullPath)).ToMarkdownList(MarkdownUtil.CodeFormatter, 10)}
+
+                    Set up a [manual build command]({buildCommandDocsUrl}).
+                    """
+                ));
+            }
+
+            // report any projects that failed to build with MSBuild
+            if (autoBuildRule.MsBuildRule.FailedProjectsOrSolutions.Any())
+            {
+                this.AddDiagnostic(new(
+                    this.Options.Language,
+                    "msbuild-build-failure",
+                    "Some projects or solutions failed to build using MSBuild",
+                    markdownMessage: $"""
+                    CodeQL was unable to build the following projects using MSBuild:
+
+                    {autoBuildRule.MsBuildRule.FailedProjectsOrSolutions.Select(p => this.MakeRelative(p.FullPath)).ToMarkdownList(MarkdownUtil.CodeFormatter, 10)}
+
+                    Set up a [manual build command]({buildCommandDocsUrl}).
+                    """
+                ));
+            }
+        }
+
         /// <summary>
         /// Gets the build strategy that the autobuilder should apply, based on the
         /// options in the `lgtm.yml` file.

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpDiagnosticClassifier.cs

@@ -0,0 +1,133 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Semmle.Autobuild.Shared;
+using Semmle.Util;
+
+namespace Semmle.Autobuild.CSharp
+{
+    /// <summary>
+    /// A diagnostic rule which tries to identify missing Xamarin SDKs.
+    /// </summary>
+    public class MissingXamarinSdkRule : DiagnosticRule
+    {
+        private const string docsUrl = "https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-xamarin-applications";
+
+        public class Result : IDiagnosticsResult
+        {
+            /// <summary>
+            /// The name of the SDK that is missing.
+            /// </summary>
+            public string SDKName { get; }
+
+            public Result(string sdkName)
+            {
+                this.SDKName = sdkName;
+            }
+
+            public DiagnosticMessage ToDiagnosticMessage<T>(Autobuilder<T> builder, DiagnosticMessage.TspSeverity? severity = null) where T : AutobuildOptionsShared => new(
+                builder.Options.Language,
+                $"missing-xamarin-{this.SDKName.ToLower()}-sdk",
+                $"Missing Xamarin SDK for {this.SDKName}",
+                severity: severity ?? DiagnosticMessage.TspSeverity.Error,
+                markdownMessage: $"[Configure your workflow]({docsUrl}) for this SDK before running CodeQL."
+            );
+        }
+
+        public MissingXamarinSdkRule() :
+            base("MSB4019:[^\"]*\"[^\"]*Xamarin\\.(?<sdkName>[^\\.]*)\\.CSharp\\.targets\"")
+        {
+        }
+
+        public override void Fire(DiagnosticClassifier classifier, Match match)
+        {
+            if (!match.Groups.TryGetValue("sdkName", out var sdkName))
+                throw new ArgumentException("Expected regular expression match to contain sdkName");
+
+            var xamarinResults = classifier.Results.OfType<Result>().Where(result =>
+                result.SDKName.Equals(sdkName.Value)
+            );
+
+            if (!xamarinResults.Any())
+                classifier.Results.Add(new Result(sdkName.Value));
+        }
+    }
+
+    public class MissingProjectFileRule : DiagnosticRule
+    {
+        private const string runsOnDocsUrl = "https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on";
+        private const string checkoutDocsUrl = "https://github.com/actions/checkout#usage";
+
+        public class Result : IDiagnosticsResult
+        {
+            /// <summary>
+            /// A set of missing project files.
+            /// </summary>
+            public HashSet<string> MissingProjectFiles { get; }
+
+            public Result()
+            {
+                this.MissingProjectFiles = new HashSet<string>();
+            }
+
+            public DiagnosticMessage ToDiagnosticMessage<T>(Autobuilder<T> builder, DiagnosticMessage.TspSeverity? severity = null) where T : AutobuildOptionsShared => new(
+                builder.Options.Language,
+                "missing-project-files",
+                "Missing project files",
+                severity: severity ?? DiagnosticMessage.TspSeverity.Warning,
+                markdownMessage: $"""
+                Some project files were not found when CodeQL built your project:
+
+                {this.MissingProjectFiles.AsEnumerable().Select(p => builder.MakeRelative(p)).ToMarkdownList(MarkdownUtil.CodeFormatter, 5)}
+
+                This may lead to subsequent failures. You can check for common causes for missing project files:
+
+                - Ensure that the project is built using the {runsOnDocsUrl.ToMarkdownLink("intended operating system")} and that filenames on case-sensitive platforms are correctly specified.
+                - If your repository uses Git submodules, ensure that those are {checkoutDocsUrl.ToMarkdownLink("checked out")} before the CodeQL action is run.
+                - If you auto-generate some project files as part of your build process, ensure that these are generated before the CodeQL action is run.
+                """
+            );
+        }
+
+        public MissingProjectFileRule() :
+            base("MSB3202: The project file \"(?<projectFile>[^\"]+)\" was not found. \\[(?<location>[^\\]]+)\\]")
+        {
+        }
+
+        public override void Fire(DiagnosticClassifier classifier, Match match)
+        {
+            if (!match.Groups.TryGetValue("projectFile", out var projectFile))
+                throw new ArgumentException("Expected regular expression match to contain projectFile");
+            if (!match.Groups.TryGetValue("location", out var location))
+                throw new ArgumentException("Expected regular expression match to contain location");
+
+            var result = classifier.Results.OfType<Result>().FirstOrDefault();
+
+            // if we do not yet have a result for this rule, create one and add it to the list
+            // of results the classifier knows about
+            if (result is null)
+            {
+                result = new Result();
+                classifier.Results.Add(result);
+            }
+
+            // then add the missing project file
+            result.MissingProjectFiles.Add(projectFile.Value);
+        }
+    }
+
+    /// <summary>
+    /// Implements a <see cref="DiagnosticClassifier" /> which applies C#-specific rules to
+    /// the build output.
+    /// </summary>
+    public class CSharpDiagnosticClassifier : DiagnosticClassifier
+    {
+        public CSharpDiagnosticClassifier()
+        {
+            // add C#-specific rules to this classifier
+            this.AddRule(new MissingXamarinSdkRule());
+            this.AddRule(new MissingProjectFileRule());
+        }
+    }
+}

csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs

@@ -15,6 +15,15 @@ namespace Semmle.Autobuild.CSharp
     /// </summary>
     internal class DotNetRule : IBuildRule<CSharpAutobuildOptions>
     {
+        public readonly List<IProjectOrSolution> FailedProjectsOrSolutions = new();
+
+        /// <summary>
+        /// A list of projects which are incompatible with DotNet.
+        /// </summary>
+        public IEnumerable<Project<CSharpAutobuildOptions>> NotDotNetProjects { get; private set; }
+
+        public DotNetRule() => NotDotNetProjects = new List<Project<CSharpAutobuildOptions>>();
+
         public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
         {
             if (!builder.ProjectsOrSolutionsToBuild.Any())
@@ -22,10 +31,12 @@ namespace Semmle.Autobuild.CSharp
 
             if (auto)
             {
-                var notDotNetProject = builder.ProjectsOrSolutionsToBuild
+                NotDotNetProjects = builder.ProjectsOrSolutionsToBuild
                     .SelectMany(p => Enumerators.Singleton(p).Concat(p.IncludedProjects))
                     .OfType<Project<CSharpAutobuildOptions>>()
-                    .FirstOrDefault(p => !p.DotNetProject);
+                    .Where(p => !p.DotNetProject);
+                var notDotNetProject = NotDotNetProjects.FirstOrDefault();
+
                 if (notDotNetProject is not null)
                 {
                     builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject);
@@ -50,7 +61,10 @@ namespace Semmle.Autobuild.CSharp
 
                         var build = GetBuildScript(builder, dotNetPath, environment, projectOrSolution.FullPath);
 
-                        ret &= BuildScript.Try(clean) & BuildScript.Try(restore) & build;
+                        ret &= BuildScript.Try(clean) & BuildScript.Try(restore) & BuildScript.OnFailure(build, ret =>
+                        {
+                            FailedProjectsOrSolutions.Add(projectOrSolution);
+                        });
                     }
                     return ret;
                 });

csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
+using Semmle.Util;
 
 namespace Semmle.Autobuild.Shared
 {

csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs

@@ -1,3 +1,4 @@
+using Semmle.Util;
 using Semmle.Util.Logging;
 using System;
 using System.Collections.Generic;
@@ -189,10 +190,11 @@ namespace Semmle.Autobuild.Shared
         /// solution file and tools.
         /// </summary>
         /// <param name="options">The command line options.</param>
-        protected Autobuilder(IBuildActions actions, TAutobuildOptions options)
+        protected Autobuilder(IBuildActions actions, TAutobuildOptions options, DiagnosticClassifier diagnosticClassifier)
         {
             Actions = actions;
             Options = options;
+            DiagnosticClassifier = diagnosticClassifier;
 
             pathsLazy = new Lazy<IEnumerable<(string, int)>>(() =>
             {
@@ -232,24 +234,53 @@ namespace Semmle.Autobuild.Shared
                 return ret ?? new List<IProjectOrSolution>();
             });
 
-            CodeQLExtractorLangRoot = Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_ROOT");
-            CodeQlPlatform = Actions.GetEnvironmentVariable("CODEQL_PLATFORM");
+            CodeQLExtractorLangRoot = Actions.GetEnvironmentVariable(EnvVars.Root(this.Options.Language));
+            CodeQlPlatform = Actions.GetEnvironmentVariable(EnvVars.Platform);
 
-            TrapDir =
-                Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_TRAP_DIR") ??
-                throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_TRAP_DIR has not been set.");
+            TrapDir = RequireEnvironmentVariable(EnvVars.TrapDir(this.Options.Language));
+            SourceArchiveDir = RequireEnvironmentVariable(EnvVars.SourceArchiveDir(this.Options.Language));
+            DiagnosticsDir = RequireEnvironmentVariable(EnvVars.DiagnosticDir(this.Options.Language));
 
-            SourceArchiveDir =
-                Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_SOURCE_ARCHIVE_DIR") ??
-                throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_SOURCE_ARCHIVE_DIR has not been set.");
+            this.diagnostics = new DiagnosticsStream(Path.Combine(DiagnosticsDir, $"autobuilder-{DateTime.UtcNow:yyyyMMddHHmm}.jsonc"));
         }
 
-        protected string TrapDir { get; }
+        /// <summary>
+        /// Retrieves the value of an environment variable named <paramref name="name"> or throws
+        /// an exception if no such environment variable has been set.
+        /// </summary>
+        /// <param name="name">The name of the environment variable.</param>
+        /// <returns>The value of the environment variable.</returns>
+        /// <exception cref="InvalidEnvironmentException">
+        /// Thrown if the environment variable is not set.
+        /// </exception>
+        protected string RequireEnvironmentVariable(string name)
+        {
+            return Actions.GetEnvironmentVariable(name) ??
+                throw new InvalidEnvironmentException($"The environment variable {name} has not been set.");
+        }
 
-        protected string SourceArchiveDir { get; }
+        public string TrapDir { get; }
+
+        public string SourceArchiveDir { get; }
+
+        public string DiagnosticsDir { get; }
+
+        protected DiagnosticClassifier DiagnosticClassifier { get; }
 
         private readonly ILogger logger = new ConsoleLogger(Verbosity.Info);
 
+        private readonly DiagnosticsStream diagnostics;
+
+        /// <summary>
+        /// Makes <see cref="path" /> relative to the root source directory.
+        /// </summary>
+        /// <param name="path">The path which to make relative.</param>
+        /// <returns>The relative path.</returns>
+        public string MakeRelative(string path)
+        {
+            return Path.GetRelativePath(this.RootDirectory, path);
+        }
+
         /// <summary>
         /// Log a given build event to the console.
         /// </summary>
@@ -260,6 +291,15 @@ namespace Semmle.Autobuild.Shared
             logger.Log(severity, format, args);
         }
 
+        /// <summary>
+        /// Write <paramref name="diagnostic"/> to the diagnostics file.
+        /// </summary>
+        /// <param name="diagnostic">The diagnostics entry to write.</param>
+        public void AddDiagnostic(DiagnosticMessage diagnostic)
+        {
+            diagnostics.AddEntry(diagnostic);
+        }
+
         /// <summary>
         /// Attempt to build this project.
         /// </summary>
@@ -283,7 +323,19 @@ namespace Semmle.Autobuild.Shared
                 Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
             }
 
-            return script.Run(Actions, startCallback, exitCallback);
+            var onOutput = BuildOutputHandler(Console.Out);
+            var onError = BuildOutputHandler(Console.Error);
+
+            var buildResult = script.Run(Actions, startCallback, exitCallback, onOutput, onError);
+
+            // if the build succeeded, all diagnostics we captured from the build output should be warnings;
+            // otherwise they should all be errors
+            var diagSeverity = buildResult == 0 ? DiagnosticMessage.TspSeverity.Warning : DiagnosticMessage.TspSeverity.Error;
+            this.DiagnosticClassifier.Results
+                .Select(result => result.ToDiagnosticMessage(this, diagSeverity))
+                .ForEach(AddDiagnostic);
+
+            return buildResult;
         }
 
         /// <summary>
@@ -291,13 +343,58 @@ namespace Semmle.Autobuild.Shared
         /// </summary>
         public abstract BuildScript GetBuildScript();
 
+
+        /// <summary>
+        /// Produces a diagnostic for the tool status page that we were unable to automatically
+        /// build the user's project and that they can manually specify a build command. This
+        /// can be overriden to implement more specific messages depending on the origin of
+        /// the failure.
+        /// </summary>
+        protected virtual void AutobuildFailureDiagnostic() => AddDiagnostic(new DiagnosticMessage(
+                this.Options.Language,
+                "autobuild-failure",
+                "Unable to build project",
+                visibility: new DiagnosticMessage.TspVisibility(statusPage: true),
+                plaintextMessage: """
+                    We were unable to automatically build your project.
+                    Set up a manual build command.
+                """
+            ));
+
+        /// <summary>
+        /// Returns a build script that can be run upon autobuild failure.
+        /// </summary>
+        /// <returns>
+        /// A build script that reports that we could not automatically detect a suitable build method.
+        /// </returns>
         protected BuildScript AutobuildFailure() =>
             BuildScript.Create(actions =>
                 {
                     Log(Severity.Error, "Could not auto-detect a suitable build method");
+
+                    AutobuildFailureDiagnostic();
+
                     return 1;
                 });
 
+        /// <summary>
+        /// Constructs a <see cref="BuildOutputHandler" /> which uses the <see cref="DiagnosticClassifier" />
+        /// to classify build output. All data also gets written to <paramref name="writer" />.
+        /// </summary>
+        /// <param name="writer">
+        /// The <see cref="TextWriter" /> to which the build output would have normally been written to.
+        /// This is normally <see cref="Console.Out" /> or <see cref="Console.Error" />.
+        /// </param>
+        /// <returns>The constructed <see cref="BuildOutputHandler" />.</returns>
+        protected BuildOutputHandler BuildOutputHandler(TextWriter writer) => new(data =>
+        {
+            if (data is not null)
+            {
+                writer.WriteLine(data);
+                DiagnosticClassifier.ClassifyLine(data);
+            }
+        });
+
         /// <summary>
         /// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
         /// </summary>

csharp/autobuilder/Semmle.Autobuild.Shared/BuildActions.cs

@@ -7,14 +7,30 @@ using System.Xml;
 using System.Net.Http;
 using System.Diagnostics.CodeAnalysis;
 using System.Threading.Tasks;
+using System.Runtime.InteropServices;
 
 namespace Semmle.Autobuild.Shared
 {
+    public delegate void BuildOutputHandler(string? data);
+
     /// <summary>
     /// Wrapper around system calls so that the build scripts can be unit-tested.
     /// </summary>
     public interface IBuildActions
     {
+
+        /// <summary>
+        /// Runs a process, captures its output, and provides it asynchronously.
+        /// </summary>
+        /// <param name="exe">The exe to run.</param>
+        /// <param name="args">The other command line arguments.</param>
+        /// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
+        /// <param name="env">Additional environment variables.</param>
+        /// <param name="onOutput">A handler for stdout output.</param>
+        /// <param name="onError">A handler for stderr output.</param>
+        /// <returns>The process exit code.</returns>
+        int RunProcess(string exe, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError);
+
         /// <summary>
         /// Runs a process and captures its output.
         /// </summary>
@@ -98,6 +114,18 @@ namespace Semmle.Autobuild.Shared
         /// </summary>
         bool IsWindows();
 
+        /// <summary>
+        /// Gets a value indicating whether we are running on macOS.
+        /// </summary>
+        /// <returns>True if we are running on macOS.</returns>
+        bool IsMacOs();
+
+        /// <summary>
+        /// Gets a value indicating whether we are running on arm.
+        /// </summary>
+        /// <returns>True if we are running on arm.</returns>
+        bool IsArm();
+
         /// <summary>
         /// Combine path segments, Path.Combine().
         /// </summary>
@@ -169,6 +197,26 @@ namespace Semmle.Autobuild.Shared
             return pi;
         }
 
+        int IBuildActions.RunProcess(string exe, string args, string? workingDirectory, System.Collections.Generic.IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
+        {
+            var pi = GetProcessStartInfo(exe, args, workingDirectory, env, true);
+            using var p = new Process
+            {
+                StartInfo = pi
+            };
+            p.StartInfo.RedirectStandardError = true;
+            p.OutputDataReceived += new DataReceivedEventHandler((sender, e) => onOutput(e.Data));
+            p.ErrorDataReceived += new DataReceivedEventHandler((sender, e) => onError(e.Data));
+
+            p.Start();
+
+            p.BeginErrorReadLine();
+            p.BeginOutputReadLine();
+
+            p.WaitForExit();
+            return p.ExitCode;
+        }
+
         int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? environment)
         {
             var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment, false);
@@ -203,6 +251,12 @@ namespace Semmle.Autobuild.Shared
 
         bool IBuildActions.IsWindows() => Win32.IsWindows();
 
+        bool IBuildActions.IsMacOs() => RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
+
+        bool IBuildActions.IsArm() =>
+            RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ||
+            RuntimeInformation.ProcessArchitecture == Architecture.Arm;
+
         string IBuildActions.PathCombine(params string[] parts) => Path.Combine(parts);
 
         void IBuildActions.WriteAllText(string filename, string contents) => File.WriteAllText(filename, contents);

csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandAutoRule.cs

@@ -11,22 +11,42 @@ namespace Semmle.Autobuild.Shared
     {
         private readonly WithDotNet<AutobuildOptionsShared> withDotNet;
 
+        /// <summary>
+        /// A list of paths to files in the project directory which we classified as scripts.
+        /// </summary>
+        public IEnumerable<string> CandidatePaths { get; private set; }
+
+        /// <summary>
+        /// The path of the script we decided to run, if any.
+        /// </summary>
+        public string? ScriptPath { get; private set; }
+
         public BuildCommandAutoRule(WithDotNet<AutobuildOptionsShared> withDotNet)
         {
             this.withDotNet = withDotNet;
+            this.CandidatePaths = new List<string>();
         }
 
+        /// <summary>
+        /// A list of extensions that we consider to be for scripts on Windows.
+        /// </summary>
         private readonly IEnumerable<string> winExtensions = new List<string> {
             ".bat",
             ".cmd",
             ".exe"
         };
 
+        /// <summary>
+        /// A list of extensions that we consider to be for scripts on Linux.
+        /// </summary>
         private readonly IEnumerable<string> linuxExtensions = new List<string> {
             "",
             ".sh"
         };
 
+        /// <summary>
+        /// A list of filenames without extensions that we think might be build scripts.
+        /// </summary>
         private readonly IEnumerable<string> buildScripts = new List<string> {
             "build"
         };
@@ -35,18 +55,25 @@ namespace Semmle.Autobuild.Shared
         {
             builder.Log(Severity.Info, "Attempting to locate build script");
 
+            // a list of extensions for files that we consider to be scripts on the current platform
             var extensions = builder.Actions.IsWindows() ? winExtensions : linuxExtensions;
+            // a list of combined base script names with the current platform's script extensions
+            // e.g. for Linux: build, build.sh
             var scripts = buildScripts.SelectMany(s => extensions.Select(e => s + e));
-            var scriptPath = builder.Paths.Where(p => scripts.Any(p.Item1.ToLower().EndsWith)).OrderBy(p => p.Item2).Select(p => p.Item1).FirstOrDefault();
+            // search through the files in the project directory for paths which end in one of
+            // the names given by `scripts`, then order them by their distance from the root
+            this.CandidatePaths = builder.Paths.Where(p => scripts.Any(p.Item1.ToLower().EndsWith)).OrderBy(p => p.Item2).Select(p => p.Item1);
+            // pick the first matching path, if there is one
+            this.ScriptPath = this.CandidatePaths.FirstOrDefault();
 
-            if (scriptPath is null)
+            if (this.ScriptPath is null)
                 return BuildScript.Failure;
 
             var chmod = new CommandBuilder(builder.Actions);
-            chmod.RunCommand("/bin/chmod", $"u+x {scriptPath}");
+            chmod.RunCommand("/bin/chmod", $"u+x {this.ScriptPath}");
             var chmodScript = builder.Actions.IsWindows() ? BuildScript.Success : BuildScript.Try(chmod.Script);
 
-            var dir = builder.Actions.GetDirectoryName(scriptPath);
+            var dir = builder.Actions.GetDirectoryName(this.ScriptPath);
 
             // A specific .NET Core version may be required
             return chmodScript & withDotNet(builder, environment =>
@@ -58,7 +85,7 @@ namespace Semmle.Autobuild.Shared
                 if (vsTools is not null)
                     command.CallBatFile(vsTools.Path);
 
-                command.RunCommand(scriptPath);
+                command.RunCommand(this.ScriptPath);
                 return command.Script;
             });
         }

csharp/autobuilder/Semmle.Autobuild.Shared/BuildScript.cs

@@ -46,6 +46,33 @@ namespace Semmle.Autobuild.Shared
         /// <returns>The exit code from this build script.</returns>
         public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout);
 
+        /// <summary>
+        /// Runs this build command.
+        /// </summary>
+        /// <param name="actions">
+        /// The interface used to implement the build actions.
+        /// </param>
+        /// <param name="startCallback">
+        /// A call back that is called every time a new process is started. The
+        /// argument to the call back is a textual representation of the process.
+        /// </param>
+        /// <param name="exitCallBack">
+        /// A call back that is called every time a new process exits. The first
+        /// argument to the call back is the exit code, and the second argument is
+        /// an exit message.
+        /// </param>
+        /// <param name="onOutput">
+        /// A handler for data read from stdout.
+        /// </param>
+        /// <param name="onError">
+        /// A handler for data read from stderr.
+        /// </param>
+        /// <returns>The exit code from this build script.</returns>
+        public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError);
+
+        /// <summary>
+        /// A build script which executes an external program or script.
+        /// </summary>
         private class BuildCommand : BuildScript
         {
             private readonly string exe, arguments;
@@ -110,8 +137,29 @@ namespace Semmle.Autobuild.Shared
                 return ret;
             }
 
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError)
+            {
+                startCallback(this.ToString(), silent);
+                var ret = 1;
+                var retMessage = "";
+                try
+                {
+                    ret = actions.RunProcess(exe, arguments, workingDirectory, environment, onOutput, onError);
+                }
+                catch (Exception ex)
+                    when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException)
+                {
+                    retMessage = ex.Message;
+                }
+                exitCallBack(ret, retMessage, silent);
+                return ret;
+            }
+
         }
 
+        /// <summary>
+        /// A build script which runs a C# function.
+        /// </summary>
         private class ReturnBuildCommand : BuildScript
         {
             private readonly Func<IBuildActions, int> func;
@@ -127,8 +175,13 @@ namespace Semmle.Autobuild.Shared
                 stdout = Array.Empty<string>();
                 return func(actions);
             }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError) => func(actions);
         }
 
+        /// <summary>
+        /// Allows two build scripts to be composed sequentially.
+        /// </summary>
         private class BindBuildScript : BuildScript
         {
             private readonly BuildScript s1;
@@ -175,6 +228,32 @@ namespace Semmle.Autobuild.Shared
                 stdout = @out;
                 return ret2;
             }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError)
+            {
+                int ret1;
+                if (s2a is not null)
+                {
+                    var stdout1 = new List<string>();
+                    var onOutputWrapper = new BuildOutputHandler(data =>
+                    {
+                        if (data is not null)
+                            stdout1.Add(data);
+
+                        onOutput(data);
+                    });
+                    ret1 = s1.Run(actions, startCallback, exitCallBack, onOutputWrapper, onError);
+                    return s2a(stdout1, ret1).Run(actions, startCallback, exitCallBack, onOutput, onError);
+                }
+
+                if (s2b is not null)
+                {
+                    ret1 = s1.Run(actions, startCallback, exitCallBack, onOutput, onError);
+                    return s2b(ret1).Run(actions, startCallback, exitCallBack, onOutput, onError);
+                }
+
+                throw new InvalidOperationException("Unexpected error");
+            }
         }
 
         /// <summary>
@@ -260,6 +339,23 @@ namespace Semmle.Autobuild.Shared
         /// </summary>
         public static BuildScript Try(BuildScript s) => s | Success;
 
+        /// <summary>
+        /// Creates a build script that runs the build script <paramref name="s" />. If
+        /// running <paramref name="s" /> fails, <paramref name="k" /> is invoked with
+        /// the exit code.
+        /// </summary>
+        /// <param name="s">The build script to run.</param>
+        /// <param name="k">
+        /// The callback that is invoked if <paramref name="s" /> failed.
+        /// </param>
+        /// <returns>The build script which implements this.</returns>
+        public static BuildScript OnFailure(BuildScript s, Action<int> k) =>
+            new BindBuildScript(s, ret => Create(actions =>
+            {
+                if (!Succeeded(ret)) k(ret);
+                return ret;
+            }));
+
         /// <summary>
         /// Creates a build script that deletes the given directory.
         /// </summary>

csharp/autobuilder/Semmle.Autobuild.Shared/DiagnosticClassifier.cs

@@ -0,0 +1,96 @@
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+using Semmle.Util;
+
+namespace Semmle.Autobuild.Shared
+{
+    /// <summary>
+    /// Direct results result from the successful application of a <see cref="DiagnosticRule" />,
+    /// which can later be converted to a corresponding <see cref="DiagnosticMessage" />.
+    /// </summary>
+    public interface IDiagnosticsResult
+    {
+        /// <summary>
+        /// Produces a <see cref="DiagnosticMessage" /> corresponding to this result.
+        /// </summary>
+        /// <param name="builder">
+        /// The autobuilder to use for constructing the base <see cref="DiagnosticMessage" />.
+        /// </param>
+        /// <param name="severity">
+        /// An optional severity value which overrides the default severity of the diagnostic.
+        /// </param>
+        /// <returns>The <see cref="DiagnosticMessage" /> corresponding to this result.</returns>
+        DiagnosticMessage ToDiagnosticMessage<T>(Autobuilder<T> builder, DiagnosticMessage.TspSeverity? severity = null) where T : AutobuildOptionsShared;
+    }
+
+    public class DiagnosticRule
+    {
+        /// <summary>
+        /// The pattern against which this rule matches build output.
+        /// </summary>
+        public Regex Pattern { get; }
+
+        /// <summary>
+        /// Constructs a diagnostic rule for the given <paramref name="pattern" />.
+        /// </summary>
+        /// <param name="pattern"></param>
+        public DiagnosticRule(Regex pattern)
+        {
+            this.Pattern = pattern;
+        }
+
+        /// <summary>
+        /// Constructs a diagnostic rule for the given regular expression <paramref name="pattern" />.
+        /// </summary>
+        /// <param name="pattern"></param>
+        public DiagnosticRule(string pattern)
+        {
+            this.Pattern = new Regex(pattern, RegexOptions.Compiled);
+        }
+
+        /// <summary>
+        /// Used by a <see cref="DiagnosticClassifier" /> <paramref name="classifier" /> to
+        /// signal that the rule has matched some build output with <paramref name="match" />.
+        /// </summary>
+        /// <param name="classifier">The classifier which is firing the rule.</param>
+        /// <param name="match">The <see cref="Match" /> that resulted from applying the rule.</param>
+        public virtual void Fire(DiagnosticClassifier classifier, Match match) { }
+    }
+
+    public class DiagnosticClassifier
+    {
+        private readonly List<DiagnosticRule> rules;
+        public readonly List<IDiagnosticsResult> Results;
+
+        public DiagnosticClassifier()
+        {
+            this.rules = new List<DiagnosticRule>();
+            this.Results = new List<IDiagnosticsResult>();
+        }
+
+        /// <summary>
+        /// Adds <paramref name="rule" /> to this classifier.
+        /// </summary>
+        /// <param name="rule">The rule to add.</param>
+        protected void AddRule(DiagnosticRule rule)
+        {
+            this.rules.Add(rule);
+        }
+
+        /// <summary>
+        /// Applies all of this classifier's rules to <paramref name="line" /> to see which match.
+        /// </summary>
+        /// <param name="line">The line to which the rules should be applied to.</param>
+        public void ClassifyLine(string line)
+        {
+            this.rules.ForEach(rule =>
+            {
+                var match = rule.Pattern.Match(line);
+                if (match.Success)
+                {
+                    rule.Fire(this, match);
+                }
+            });
+        }
+    }
+}

csharp/autobuilder/Semmle.Autobuild.Shared/EnvVars.cs

@@ -0,0 +1,13 @@
+using Semmle.Util;
+
+namespace Semmle.Autobuild.Shared
+{
+    public static class EnvVars
+    {
+        public const string Platform = "CODEQL_PLATFORM";
+        public static string Root(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_ROOT";
+        public static string TrapDir(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_TRAP_DIR";
+        public static string SourceArchiveDir(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_SOURCE_ARCHIVE_DIR";
+        public static string DiagnosticDir(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_DIAGNOSTIC_DIR";
+    }
+}

csharp/autobuilder/Semmle.Autobuild.Shared/MarkdownUtil.cs

@@ -0,0 +1,65 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Semmle.Autobuild.Shared
+{
+    public static class MarkdownUtil
+    {
+        /// <summary>
+        /// Formats items as markdown inline code.
+        /// </summary>
+        /// <returns>A function which formats items as markdown inline code.</returns>
+        public static readonly Func<string, string> CodeFormatter = item => $"`{item}`";
+
+        /// <summary>
+        /// Formats the string as a markdown link.
+        /// </summary>
+        /// <param name="link">The URL for the link.</param>
+        /// <param name="title">The text that is displayed.</param>
+        /// <returns>A string containing a markdown-formatted link.</returns>
+        public static string ToMarkdownLink(this string link, string title) => $"[{title}]({link})";
+
+        /// <summary>
+        /// Renders <see cref="projects" /> as a markdown list of the project paths.
+        /// </summary>
+        /// <param name="projects">
+        /// The list of projects whose paths should be rendered as a markdown list.
+        /// </param>
+        /// <param name="limit">The maximum number of items to include in the list.</param>
+        /// <returns>Returns the markdown list as a string.</returns>
+        public static string ToMarkdownList(this IEnumerable<IProjectOrSolution> projects, int? limit = null)
+        {
+            return projects.ToMarkdownList(p => $"`{p.FullPath}`", limit);
+        }
+
+        /// <summary>
+        /// Renders <see cref="items" /> as a markdown list.
+        /// </summary>
+        /// <typeparam name="T">The item type.</typeparam>
+        /// <param name="items">The list that should be formatted as a markdown list.</param>
+        /// <param name="formatter">A function which converts individual items into a string representation.</param>
+        /// <param name="limit">The maximum number of items to include in the list.</param>
+        /// <returns>Returns the markdown list as a string.</returns>
+        public static string ToMarkdownList<T>(this IEnumerable<T> items, Func<T, string> formatter, int? limit = null)
+        {
+            var sb = new StringBuilder();
+
+            // if there is a limit, take at most that many items from the  start of the list
+            var list = limit is not null ? items.Take(limit.Value) : items;
+            sb.Append(string.Join('\n', list.Select(item => $"- {formatter(item)}")));
+
+            // if there were more items than allowed in the list, add an extra item indicating
+            // how many more items there were
+            var length = items.Count();
+
+            if (limit is not null && length > limit)
+            {
+                sb.Append($"\n- and {length - limit} more. View the CodeQL logs for a full list.");
+            }
+
+            return sb.ToString();
+        }
+    }
+}

csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs

@@ -1,17 +1,39 @@
 using Semmle.Util.Logging;
+using System.Collections.Generic;
 using System.Linq;
 
 namespace Semmle.Autobuild.Shared
 {
+    internal static class MsBuildCommandExtensions
+    {
+        /// <summary>
+        /// Appends a call to msbuild.
+        /// </summary>
+        /// <param name="cmdBuilder"></param>
+        /// <param name="builder"></param>
+        /// <returns></returns>
+        public static CommandBuilder MsBuildCommand(this CommandBuilder cmdBuilder, IAutobuilder<AutobuildOptionsShared> builder)
+        {
+            var isArmMac = builder.Actions.IsMacOs() && builder.Actions.IsArm();
+
+            // mono doesn't ship with `msbuild` on Arm-based Macs, but we can fall back to
+            // msbuild that ships with `dotnet` which can be invoked with `dotnet msbuild`
+            // perhaps we should do this on all platforms?
+            return isArmMac ?
+                cmdBuilder.RunCommand("dotnet").Argument("msbuild") :
+                cmdBuilder.RunCommand("msbuild");
+        }
+    }
+
     /// <summary>
     /// A build rule using msbuild.
     /// </summary>
     public class MsBuildRule : IBuildRule<AutobuildOptionsShared>
     {
         /// <summary>
-        /// The name of the msbuild command.
+        /// A list of solutions or projects which failed to build.
         /// </summary>
-        private const string msBuild = "msbuild";
+        public readonly List<IProjectOrSolution> FailedProjectsOrSolutions = new();
 
         public BuildScript Analyse(IAutobuilder<AutobuildOptionsShared> builder, bool auto)
         {
@@ -57,7 +79,7 @@ namespace Semmle.Autobuild.Shared
                             Script;
                     var nugetRestore = GetNugetRestoreScript();
                     var msbuildRestoreCommand = new CommandBuilder(builder.Actions).
-                        RunCommand(msBuild).
+                        MsBuildCommand(builder).
                         Argument("/t:restore").
                         QuoteArgument(projectOrSolution.FullPath);
 
@@ -95,7 +117,7 @@ namespace Semmle.Autobuild.Shared
                     command.RunCommand("set Platform=&& type NUL", quoteExe: false);
                 }
 
-                command.RunCommand(msBuild);
+                command.MsBuildCommand(builder);
                 command.QuoteArgument(projectOrSolution.FullPath);
 
                 var target = builder.Options.MsBuildTarget ?? "rebuild";
@@ -110,7 +132,13 @@ namespace Semmle.Autobuild.Shared
 
                 command.Argument(builder.Options.MsBuildArguments);
 
-                ret &= command.Script;
+                // append the build script which invokes msbuild to the overall build script `ret`;
+                // we insert a check that building the current project or solution was successful:
+                // if it was not successful, we add it to `FailedProjectsOrSolutions`
+                ret &= BuildScript.OnFailure(command.Script, ret =>
+                {
+                    FailedProjectsOrSolutions.Add(projectOrSolution);
+                });
             }
 
             return ret;

csharp/autobuilder/Semmle.Autobuild.Shared/ProjectOrSolution.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using Semmle.Util;
 
 namespace Semmle.Autobuild.Shared
 {

csharp/downgrades/cd877b8cc2fb8327499f96fbefd01bb988b2ed63/upgrade.properties

@@ -0,0 +1,3 @@
+description: Remove a relation for scoped annotations.
+compatibility: backwards
+scoped_annotation.rel: delete

csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs

@@ -106,36 +106,6 @@ namespace Semmle.BuildAnalyser
                 return result;
             }
 
-            // Attempt to load the reference from the GAC.
-            try
-            {
-                var loadedAssembly = System.Reflection.Assembly.ReflectionOnlyLoad(id);
-
-                if (loadedAssembly is not null)
-                {
-                    // The assembly was somewhere we haven't indexed before.
-                    // Add this assembly to our index so that subsequent lookups are faster.
-
-                    result = AssemblyInfo.MakeFromAssembly(loadedAssembly);
-                    assemblyInfoById[id] = result;
-                    assemblyInfoByFileName[loadedAssembly.Location] = result;
-                    return result;
-                }
-            }
-            catch (FileNotFoundException)
-            {
-                // A suitable assembly could not be found
-            }
-            catch (FileLoadException)
-            {
-                // The assembly cannot be loaded for some reason
-                // e.g. The name is malformed.
-            }
-            catch (PlatformNotSupportedException)
-            {
-                // .NET Core does not have a GAC.
-            }
-
             // Fallback position - locate the assembly by its lower-case name only.
             var asmName = assemblyName.ToLowerInvariant();
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedSymbol.cs

@@ -59,6 +59,19 @@ namespace Semmle.Extraction.CSharp.Entities
             }
         }
 
+        protected void PopulateScopedKind(TextWriter trapFile, ScopedKind kind)
+        {
+            switch (kind)
+            {
+                case ScopedKind.ScopedRef:
+                    trapFile.scoped_annotation(this, Kinds.ScopedAnnotation.ScopedRef);
+                    break;
+                case ScopedKind.ScopedValue:
+                    trapFile.scoped_annotation(this, Kinds.ScopedAnnotation.ScopedValue);
+                    break;
+            }
+        }
+
         protected void ExtractCompilerGenerated(TextWriter trapFile)
         {
             if (Symbol.IsImplicitlyDeclared)

csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs

@@ -33,7 +33,8 @@ namespace Semmle.Extraction.CSharp.Entities
             PopulateRefKind(trapFile, Symbol.RefKind);
 
             var unboundFieldKey = Field.Create(Context, Symbol.OriginalDefinition);
-            trapFile.fields(this, (Symbol.IsConst ? 2 : 1), Symbol.Name, ContainingType, Type.TypeRef, unboundFieldKey);
+            var kind = Symbol.IsConst ? VariableKind.Const : VariableKind.None;
+            trapFile.fields(this, kind, Symbol.Name, ContainingType, Type.TypeRef, unboundFieldKey);
 
             PopulateModifiers(trapFile);
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs

@@ -23,17 +23,23 @@ namespace Semmle.Extraction.CSharp.Entities
         public void PopulateManual(Expression parent, bool isVar)
         {
             var trapFile = Context.TrapWriter.Writer;
-            var (kind, type) = Symbol is ILocalSymbol l
-                ? (l.IsRef ? 3 : l.IsConst ? 2 : 1, l.GetAnnotatedType())
-                : (1, parent.Type);
-            trapFile.localvars(this, kind, Symbol.Name, isVar ? 1 : 0, Type.Create(Context, type).TypeRef, parent);
+            var @var = isVar ? 1 : 0;
 
             if (Symbol is ILocalSymbol local)
             {
+                var kind = local.IsRef ? Kinds.VariableKind.Ref : local.IsConst ? Kinds.VariableKind.Const : Kinds.VariableKind.None;
+                var type = local.GetAnnotatedType();
+                trapFile.localvars(this, kind, Symbol.Name, @var, Type.Create(Context, type).TypeRef, parent);
+
                 PopulateNullability(trapFile, local.GetAnnotatedType());
+                PopulateScopedKind(trapFile, local.ScopedKind);
                 if (local.IsRef)
                     trapFile.type_annotation(this, Kinds.TypeAnnotation.Ref);
             }
+            else
+            {
+                trapFile.localvars(this, Kinds.VariableKind.None, Symbol.Name, @var, Type.Create(Context, parent.Type).TypeRef, parent);
+            }
 
             trapFile.localvar_location(this, Location);
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs

@@ -100,6 +100,7 @@ namespace Semmle.Extraction.CSharp.Entities
             PopulateAttributes();
             PopulateNullability(trapFile, Symbol.GetAnnotatedType());
             PopulateRefKind(trapFile, Symbol.RefKind);
+            PopulateScopedKind(trapFile, Symbol.ScopedKind);
 
             if (Symbol.Name != Original.Symbol.Name)
                 Context.ModelError(Symbol, "Inconsistent parameter declaration");

csharp/extractor/Semmle.Extraction.CSharp/Kinds/ScopedAnnotation.cs

@@ -0,0 +1,8 @@
+namespace Semmle.Extraction.Kinds;
+
+public enum ScopedAnnotation
+{
+    None = 0,
+    ScopedRef = 1,
+    ScopedValue = 2
+}

csharp/extractor/Semmle.Extraction.CSharp/Kinds/VariableKind.cs

@@ -0,0 +1,8 @@
+namespace Semmle.Extraction.Kinds;
+
+public enum VariableKind
+{
+    None = 1,
+    Const = 2,
+    Ref = 3
+}

csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs

@@ -191,8 +191,8 @@ namespace Semmle.Extraction.CSharp
         internal static void field_location(this TextWriter trapFile, Field field, Location location) =>
             trapFile.WriteTuple("field_location", field, location);
 
-        internal static void fields(this TextWriter trapFile, Field field, int @const, string name, Type declaringType, Type fieldType, Field unboundKey) =>
-            trapFile.WriteTuple("fields", field, @const, name, declaringType, fieldType, unboundKey);
+        internal static void fields(this TextWriter trapFile, Field field, VariableKind kind, string name, Type declaringType, Type fieldType, Field unboundKey) =>
+            trapFile.WriteTuple("fields", field, (int)kind, name, declaringType, fieldType, unboundKey);
 
         internal static void general_type_parameter_constraints(this TextWriter trapFile, TypeParameterConstraints constraints, int hasKind) =>
             trapFile.WriteTuple("general_type_parameter_constraints", constraints, hasKind);
@@ -227,8 +227,8 @@ namespace Semmle.Extraction.CSharp
         internal static void localvar_location(this TextWriter trapFile, LocalVariable var, Location location) =>
             trapFile.WriteTuple("localvar_location", var, location);
 
-        internal static void localvars(this TextWriter trapFile, LocalVariable key, int @const, string name, int @var, Type type, Expression expr) =>
-            trapFile.WriteTuple("localvars", key, @const, name, @var, type, expr);
+        internal static void localvars(this TextWriter trapFile, LocalVariable key, VariableKind kind, string name, int @var, Type type, Expression expr) =>
+            trapFile.WriteTuple("localvars", key, (int)kind, name, @var, type, expr);
 
         public static void metadata_handle(this TextWriter trapFile, IEntity entity, Location assembly, int handleValue) =>
             trapFile.WriteTuple("metadata_handle", entity, assembly, handleValue);
@@ -462,5 +462,8 @@ namespace Semmle.Extraction.CSharp
 
         internal static void file_extraction_mode(this System.IO.TextWriter trapFile, Entities.File file, ExtractorMode mode) =>
             trapFile.WriteTuple("file_extraction_mode", file, mode);
+
+        internal static void scoped_annotation(this TextWriter trapFile, IEntity element, ScopedAnnotation @scoped) =>
+            trapFile.WriteTuple("scoped_annotation", element, (int)@scoped);
     }
 }

csharp/extractor/Semmle.Extraction/TrapWriter.cs

@@ -223,7 +223,7 @@ namespace Semmle.Extraction
             {
                 FileUtils.MoveOrReplace(tmpSrcFile, dest);
             }
-            catch (IOException ex)
+            catch (Exception ex)
             {
                 // If this happened, it was probably because the same file was compiled multiple times.
                 // In any case, this is not a fatal error.

csharp/extractor/Semmle.Util/Language.cs

@@ -1,4 +1,4 @@
-namespace Semmle.Autobuild.Shared
+namespace Semmle.Util
 {
     public sealed class Language
     {

csharp/extractor/Semmle.Util/Semmle.Util.csproj

@@ -15,6 +15,7 @@
 
   <ItemGroup>
     <PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
   </ItemGroup>
 
 </Project>

csharp/extractor/Semmle.Util/ToolStatusPage.cs

@@ -0,0 +1,224 @@
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+using Newtonsoft.Json.Serialization;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// Represents diagnostic messages for the tool status page.
+    /// </summary>
+    public class DiagnosticMessage
+    {
+        /// <summary>
+        /// Represents sources of diagnostic messages.
+        /// </summary>
+        public class TspSource
+        {
+            /// <summary>
+            /// An identifier under which it makes sense to group this diagnostic message.
+            /// This is used to build the SARIF reporting descriptor object.
+            /// </summary>
+            public string Id { get; }
+            /// <summary>
+            /// Display name for the ID. This is used to build the SARIF reporting descriptor object.
+            /// </summary>
+            public string Name { get; }
+            /// <summary>
+            /// Name of the CodeQL extractor. This is used to identify which tool component the reporting descriptor object should be nested under in SARIF.
+            /// </summary>
+            public string? ExtractorName { get; }
+
+            public TspSource(string id, string name, string? extractorName = null)
+            {
+                Id = id;
+                Name = name;
+                ExtractorName = extractorName;
+            }
+        }
+
+        /// <summary>
+        /// Enumerates severity levels for diagnostics.
+        /// </summary>
+        [JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]
+        public enum TspSeverity
+        {
+            Note,
+            Warning,
+            Error
+        }
+
+        /// <summary>
+        /// Stores flags indicating where the diagnostic should be displayed.
+        /// </summary>
+        public class TspVisibility
+        {
+            /// <summary>
+            /// True if the message should be displayed on the status page (defaults to false).
+            /// </summary>
+            public bool? StatusPage { get; }
+            /// <summary>
+            /// True if the message should be counted in the diagnostics summary table printed by
+            /// <c>codeql database analyze</c> (defaults to false).
+            /// </summary>
+            public bool? CLISummaryTable { get; }
+            /// <summary>
+            /// True if the message should be sent to telemetry (defaults to false).
+            /// </summary>
+            public bool? Telemetry { get; }
+
+            public TspVisibility(bool? statusPage = null, bool? cliSummaryTable = null, bool? telemetry = null)
+            {
+                this.StatusPage = statusPage;
+                this.CLISummaryTable = cliSummaryTable;
+                this.Telemetry = telemetry;
+            }
+        }
+
+        /// <summary>
+        /// Represents source code locations for diagnostic messages.
+        /// </summary>
+        public class TspLocation
+        {
+            /// <summary>
+            /// Path to the affected file if appropriate, relative to the source root.
+            /// </summary>
+            public string? File { get; }
+            /// <summary>
+            /// The line where the range to which the diagnostic relates to starts.
+            /// </summary>
+            public int? StartLine { get; }
+            /// <summary>
+            /// The column where the range to which the diagnostic relates to starts.
+            /// </summary>
+            public int? StartColumn { get; }
+            /// <summary>
+            /// The line where the range to which the diagnostic relates to ends.
+            /// </summary>
+            public int? EndLine { get; }
+            /// <summary>
+            /// The column where the range to which the diagnostic relates to ends.
+            /// </summary>
+            public int? EndColumn { get; }
+
+            public TspLocation(string? file = null, int? startLine = null, int? startColumn = null, int? endLine = null, int? endColumn = null)
+            {
+                this.File = file;
+                this.StartLine = startLine;
+                this.StartColumn = startColumn;
+                this.EndLine = endLine;
+                this.EndColumn = endColumn;
+            }
+        }
+
+        /// <summary>
+        /// ISO 8601 timestamp.
+        /// </summary>
+        public string Timestamp { get; }
+        /// <summary>
+        /// The source of the diagnostic message.
+        /// </summary>
+        public TspSource Source { get; }
+        /// <summary>
+        /// GitHub flavored Markdown formatted message. Should include inline links to any help pages.
+        /// </summary>
+        public string? MarkdownMessage { get; }
+        /// <summary>
+        /// Plain text message. Used by components where the string processing needed to support
+        /// Markdown is cumbersome.
+        /// </summary>
+        public string? PlaintextMessage { get; }
+        /// <summary>
+        /// List of help links intended to supplement <see cref="PlaintextMessage" />.
+        /// </summary>
+        public List<string> HelpLinks { get; }
+        /// <summary>
+        /// SARIF severity.
+        /// </summary>
+        public TspSeverity? Severity { get; }
+        /// <summary>
+        /// If true, then this message won't be presented to users.
+        /// </summary>
+        public bool Internal { get; }
+        public TspVisibility Visibility { get; }
+        public TspLocation Location { get; }
+        /// <summary>
+        /// Structured metadata about the diagnostic message.
+        /// </summary>
+        public Dictionary<string, object> Attributes { get; }
+
+        public DiagnosticMessage(
+            Language language, string id, string name, string? markdownMessage = null, string? plaintextMessage = null,
+            TspVisibility? visibility = null, TspLocation? location = null, TspSeverity? severity = TspSeverity.Error,
+            DateTime? timestamp = null, bool? intrnl = null
+        )
+        {
+            this.Source = new TspSource(
+                id: $"{language.UpperCaseName.ToLower()}/autobuilder/{id}",
+                name: name,
+                extractorName: language.UpperCaseName.ToLower()
+            );
+            this.Timestamp = (timestamp ?? DateTime.UtcNow).ToString("o", CultureInfo.InvariantCulture);
+            this.HelpLinks = new List<string>();
+            this.Attributes = new Dictionary<string, object>();
+            this.Severity = severity;
+            this.Visibility = visibility ?? new TspVisibility(statusPage: true);
+            this.Location = location ?? new TspLocation();
+            this.Internal = intrnl ?? false;
+            this.MarkdownMessage = markdownMessage;
+            this.PlaintextMessage = plaintextMessage;
+        }
+    }
+
+    /// <summary>
+    /// A wrapper around an underlying <see cref="StreamWriter" /> which allows
+    /// <see cref="DiagnosticMessage" /> objects to be serialized to it.
+    /// </summary>
+    public sealed class DiagnosticsStream : IDisposable
+    {
+        private readonly JsonSerializer serializer;
+        private readonly StreamWriter writer;
+
+        /// <summary>
+        /// Initialises a new <see cref="DiagnosticsStream" /> for a file at <paramref name="path" />.
+        /// </summary>
+        /// <param name="path">The path to the file that should be created.</param>
+        public DiagnosticsStream(string path)
+        {
+            this.writer = File.CreateText(path);
+
+            var contractResolver = new DefaultContractResolver
+            {
+                NamingStrategy = new CamelCaseNamingStrategy()
+            };
+
+            serializer = new JsonSerializer
+            {
+                ContractResolver = contractResolver,
+                NullValueHandling = NullValueHandling.Ignore
+            };
+        }
+
+        /// <summary>
+        /// Adds <paramref name="message" /> as a new diagnostics entry.
+        /// </summary>
+        /// <param name="message">The diagnostics entry to add.</param>
+        public void AddEntry(DiagnosticMessage message)
+        {
+            serializer.Serialize(writer, message);
+            writer.Flush();
+        }
+
+        /// <summary>
+        /// Releases all resources used by the <see cref="DiagnosticsStream" /> object.
+        /// </summary>
+        public void Dispose()
+        {
+            writer.Dispose();
+        }
+    }
+}

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.4.2
+version: 1.4.3-dev
 groups:
     - csharp
     - solorigate

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.4.2
+version: 1.4.3-dev
 groups:
     - csharp
     - solorigate

csharp/ql/integration-tests/all-platforms/diag_dotnet_incompatible/diagnostics.expected

@@ -0,0 +1,32 @@
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL found some projects which cannot be built with .NET Core:\n\n- `test.csproj`",
+  "severity": "warning",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/dotnet-incompatible-projects",
+    "name": "Some projects are incompatible with .NET Core"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL was unable to build the following projects using MSBuild:\n\n- `test.csproj`\n\nSet up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/msbuild-build-failure",
+    "name": "Some projects or solutions failed to build using MSBuild"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}

csharp/ql/integration-tests/all-platforms/diag_dotnet_incompatible/test.csproj

@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>Example</RootNamespace>
+    <AssemblyName>Example</AssemblyName>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <TargetFrameworkProfile />
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x86'">
+    <DebugSymbols>true</DebugSymbols>
+    <OutputPath>bin\x86\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <DebugType>full</DebugType>
+    <PlatformTarget>x86</PlatformTarget>
+    <ErrorReport>prompt</ErrorReport>
+    <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x86'">
+    <OutputPath>bin\x86\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <Optimize>true</Optimize>
+    <DebugType>pdbonly</DebugType>
+    <PlatformTarget>x86</PlatformTarget>
+    <ErrorReport>prompt</ErrorReport>
+    <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Program.cs" />
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>

csharp/ql/integration-tests/all-platforms/diag_dotnet_incompatible/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
+check_diagnostics()

csharp/ql/integration-tests/all-platforms/diag_missing_project_files/diagnostics.expected

@@ -0,0 +1,32 @@
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL was unable to build the following projects using MSBuild:\n\n- `test.sln`\n\nSet up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/msbuild-build-failure",
+    "name": "Some projects or solutions failed to build using MSBuild"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "Some project files were not found when CodeQL built your project:\n\n- `Example.csproj`\n- `Example.Test.csproj`\n\nThis may lead to subsequent failures. You can check for common causes for missing project files:\n\n- Ensure that the project is built using the [intended operating system](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on) and that filenames on case-sensitive platforms are correctly specified.\n- If your repository uses Git submodules, ensure that those are [checked out](https://github.com/actions/checkout#usage) before the CodeQL action is run.\n- If you auto-generate some project files as part of your build process, ensure that these are generated before the CodeQL action is run.",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/missing-project-files",
+    "name": "Missing project files"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}

csharp/ql/integration-tests/all-platforms/diag_missing_project_files/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
+check_diagnostics()

csharp/ql/integration-tests/all-platforms/diag_missing_project_files/test.sln

@@ -0,0 +1,26 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2012
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example", "Example.csproj", "{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example.Test", "Example.Test.csproj", "{F4587B5F-9918-4079-9291-5A08CD9761FB}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x86 = Debug|x86
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}.Debug|x86.ActiveCfg = Debug|x86
+		{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}.Debug|x86.Build.0 = Debug|x86
+		{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}.Release|x86.ActiveCfg = Release|x86
+		{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}.Release|x86.Build.0 = Release|x86
+		{F4587B5F-9918-4079-9291-5A08CD9761FB}.Debug|x86.ActiveCfg = Debug|x86
+		{F4587B5F-9918-4079-9291-5A08CD9761FB}.Debug|x86.Build.0 = Debug|x86
+		{F4587B5F-9918-4079-9291-5A08CD9761FB}.Release|x86.ActiveCfg = Release|x86
+		{F4587B5F-9918-4079-9291-5A08CD9761FB}.Release|x86.Build.0 = Release|x86
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

csharp/ql/integration-tests/all-platforms/diag_missing_xamarin_sdk/diagnostics.expected

@@ -0,0 +1,48 @@
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL was unable to build the following projects using .NET Core:\n\n- `test.csproj`\n\nSet up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/dotnet-build-failure",
+    "name": "Some projects or solutions failed to build using .NET Core"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL was unable to build the following projects using MSBuild:\n\n- `test.csproj`\n\nSet up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/msbuild-build-failure",
+    "name": "Some projects or solutions failed to build using MSBuild"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "[Configure your workflow](https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-xamarin-applications) for this SDK before running CodeQL.",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/missing-xamarin-ios-sdk",
+    "name": "Missing Xamarin SDK for iOS"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}

csharp/ql/integration-tests/all-platforms/diag_missing_xamarin_sdk/test.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <LanguageTargets>$(MSBuildExtensionsPath)\Xamarin\iOS\Xamarin.iOS.CSharp.targets</LanguageTargets>
+  </PropertyGroup>
+
+</Project>

csharp/ql/integration-tests/all-platforms/diag_missing_xamarin_sdk/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
+check_diagnostics()

csharp/ql/integration-tests/all-platforms/dotnet_publish/test.py

@@ -1,8 +1,9 @@
 import os
 from create_database_utils import *
 
-run_codeql_database_create(['dotnet publish'], test_db="default-db", db=None, lang="csharp")
+artifacts = 'bin/Temp'
+run_codeql_database_create([f"dotnet publish -o {artifacts}"], test_db="default-db", db=None, lang="csharp")
 
 ## Check that the publish folder is created.
-if not os.path.isdir("bin/Debug/net7.0/publish/"):
+if not os.path.isdir(artifacts):
     raise Exception("The publish artifact folder was not created.") 

csharp/ql/integration-tests/all-platforms/msbuild/Program.cs

@@ -0,0 +1,13 @@
+using System;
+
+namespace Test
+{
+    public class Program
+    {
+        public static int Main(string[] args)
+        {
+            Console.WriteLine("Hello world!");
+            return 0;
+        }
+    }
+}

csharp/ql/integration-tests/all-platforms/msbuild/test.csproj

@@ -0,0 +1,7 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net4.0</TargetFramework>
+  </PropertyGroup>
+</Project>

csharp/ql/integration-tests/all-platforms/msbuild/test.py

@@ -0,0 +1,4 @@
+from create_database_utils import *
+
+# force CodeQL to use MSBuild by setting `LGTM_INDEX_MSBUILD_TARGET`
+run_codeql_database_create([], test_db="default-db", db=None, lang="csharp", extra_env={ 'LGTM_INDEX_MSBUILD_TARGET': 'Build' })

csharp/ql/integration-tests/posix-only/diag_autobuild_script/build.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exit 1

csharp/ql/integration-tests/posix-only/diag_autobuild_script/diagnostics.expected

@@ -0,0 +1,32 @@
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL attempted to build your project using a script located at `build.sh`, which failed. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/script-failure",
+    "name": "Unable to build project using build script"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL could not find any project or solution files in your repository. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/no-projects-or-solutions",
+    "name": "No project or solutions files found"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}

csharp/ql/integration-tests/posix-only/diag_autobuild_script/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
+check_diagnostics()

csharp/ql/integration-tests/posix-only/diag_multiple_scripts/build.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exit 1

csharp/ql/integration-tests/posix-only/diag_multiple_scripts/diagnostics.expected

@@ -0,0 +1,32 @@
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL could not find any project or solution files in your repository. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/no-projects-or-solutions",
+    "name": "No project or solutions files found"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL found multiple potential build scripts for your project and attempted to run `build.sh`, which failed. This may not be the right build script for your project. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/multiple-build-scripts",
+    "name": "There are multiple potential build scripts"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}

csharp/ql/integration-tests/posix-only/diag_multiple_scripts/scripts/build.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exit 1

csharp/ql/integration-tests/posix-only/diag_multiple_scripts/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
+check_diagnostics()

csharp/ql/integration-tests/windows-only/diag_autobuild_script/build.bat

@@ -0,0 +1 @@
+exit /b 1

csharp/ql/integration-tests/windows-only/diag_autobuild_script/diagnostics.expected

@@ -0,0 +1,32 @@
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL attempted to build your project using a script located at `build.bat`, which failed. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/script-failure",
+    "name": "Unable to build project using build script"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL could not find any project or solution files in your repository. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/no-projects-or-solutions",
+    "name": "No project or solutions files found"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}

csharp/ql/integration-tests/windows-only/diag_autobuild_script/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
+check_diagnostics()

csharp/ql/integration-tests/windows-only/diag_multiple_scripts/build.bat

@@ -0,0 +1 @@
+exit /b 1

csharp/ql/integration-tests/windows-only/diag_multiple_scripts/diagnostics.expected

@@ -0,0 +1,32 @@
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL could not find any project or solution files in your repository. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/no-projects-or-solutions",
+    "name": "No project or solutions files found"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}
+{
+  "attributes": {},
+  "helpLinks": [],
+  "internal": false,
+  "location": {},
+  "markdownMessage": "CodeQL found multiple potential build scripts for your project and attempted to run `build.bat`, which failed. This may not be the right build script for your project. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/multiple-build-scripts",
+    "name": "There are multiple potential build scripts"
+  },
+  "visibility": {
+    "statusPage": true
+  }
+}

csharp/ql/integration-tests/windows-only/diag_multiple_scripts/scripts/build.bat

@@ -0,0 +1 @@
+exit /b 1

csharp/ql/integration-tests/windows-only/diag_multiple_scripts/test.py

@@ -0,0 +1,5 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
+check_diagnostics()

csharp/ql/lib/change-notes/2023-02-07-scoped-modifier.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* C# 11: Added extractor support for the `scoped` modifier annotation on parameters and local variables.

csharp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-all
-version: 0.5.2
+version: 0.5.3-dev
 groups: csharp
 dbscheme: semmlecode.csharp.dbscheme
 extractor: csharp

csharp/ql/lib/semmle/code/csharp/Variable.qll

@@ -71,6 +71,11 @@ class LocalScopeVariable extends Variable, @local_scope_variable {
    */
   predicate isRef() { none() }
 
+  /**
+   * Holds if this local variable or parameter is `scoped`.
+   */
+  predicate isScoped() { scoped_annotation(this, _) }
+
   override predicate hasQualifiedName(string qualifier, string name) { none() }
 }
 

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl2.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;
 

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,13 +862,8 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
@@ -831,6 +871,12 @@ private module Stage1 implements StageSig {
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {
@@ -1223,7 +1269,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1526,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2125,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2394,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2474,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2808,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;