wip

.git-blame-ignore-revs

@@ -1,21 +0,0 @@
-# .git-blame-ignore-revs
-# Auto-formatted Java
-730eae952139209fe9fdf598541d608f4c0c0c84
-# Auto-formatted C#
-5ad7ed49dd3de03ec6dcfcb6848758a6a987e11c
-# Auto-formatted C/C++
-ef97e539ec1971494d4bba5cafe82e00bc8217ac
-# Auto-formatted Python
-21d5fa836b3a7d020ba45e8b8168b145a9772131
-# Auto-formatted JavaScript
-8d97fe9ed327a9546ff2eaf515cf0f5214deddd9
-# Auto-formatted Ruby
-a5d229903d2f12d45f2c2c38822f1d0e7504ae7f
-# Auto-formatted Go
-08c658e66bf867090033ea096e244a93d46c0aa7
-# Auto-formatted Swift
-711d7057f79fb7d72fc3b35e010bd018f9009169
-# Auto-formatted shared ql packs
-3640b6d3a8ce9edf8e1d3ed106fe8526cf255bc0
-# Auto-formatted taint tracking files
-159d8e978c51959b380838c080d891b66e763b19

.github/ISSUE_TEMPLATE/lgtm-com---false-positive.md

@@ -0,0 +1,24 @@
+---
+name: LGTM.com - false positive
+about: Tell us about an alert that shouldn't be reported
+title: LGTM.com - false positive
+labels: false-positive
+assignees: ''
+
+---
+
+**Description of the false positive**
+
+<!-- Please explain briefly why you think it shouldn't be included. -->
+
+**URL to the alert on the project page on LGTM.com**
+
+<!--
+1. Open the project on LGTM.com.
+For example, https://lgtm.com/projects/g/pallets/click/.
+2. Switch to the `Alerts` tab. For example, https://lgtm.com/projects/g/pallets/click/alerts/.
+3. Scroll to the alert that you would like to report.
+4. Click on the right most icon `View this alert within the complete file`.
+5. A new browser tab opens. Copy and paste the page URL here.
+For example, https://lgtm.com/projects/g/pallets/click/snapshot/719fb7d8322b0767cdd1e5903ba3eb3233ba8dd5/files/click/_winconsole.py#xa08d213ab3289f87:1.
+-->

.github/ISSUE_TEMPLATE/ql---general.md

@@ -10,5 +10,5 @@ assignees: ''
 **Description of the issue**
 
 <!-- Please explain briefly what is the problem.
-If it is about a GitHub project, please include its URL. -->
+If it is about an LGTM project, please include its URL.-->
 

.github/ISSUE_TEMPLATE/ql--false-positive.md

@@ -1,36 +0,0 @@
----
-name: CodeQL false positive
-about: Report CodeQL alerts that you think should not have been detected (not applicable, not exploitable, etc.)
-title: False positive
-labels: false-positive
-assignees: ''
-
----
-
-**Description of the false positive**
-
-<!-- Please explain briefly why you think it shouldn't be included. -->
-
-**Code samples or links to source code**
-
-<!--
-For open source code: file links with line numbers on GitHub, for example:
-https://github.com/github/codeql/blob/dc440aaee6695deb0d9676b87e06ea984e1b4ae5/javascript/ql/test/query-tests/Security/CWE-078/CommandInjection/exec-sh2.js#L10
-
-For closed source code: (redacted) code samples that illustrate the problem, for example:
-
-```
-function execSh(command, options) {
-    return cp.spawn(getShell(), ["-c", command], options) // <- command line injection
-};
-```
--->
-
-**URL to the alert on GitHub code scanning (optional)**
-
-<!--
-1. Open the project on GitHub.com.
-2. Switch to the `Security` tab.
-3. Browse to the alert that you would like to report.
-4. Copy and paste the page URL here.
--->

.github/actions/cache-query-compilation/action.yml

@@ -1,149 +0,0 @@
-name: Cache query compilation
-description: Caches CodeQL compilation caches - should be run both on PRs and pushes to main.
-
-inputs:
-  key:
-    description: 'The cache key to use - should be unique to the workflow'
-    required: true
-
-outputs:
-  cache-dir:
-    description: "The directory where the cache was stored"
-    value: ${{ steps.output-compilation-dir.outputs.compdir }}
-
-runs:
-  using: composite
-  steps:
-    # calculate the merge-base with main, in a way that works both on PRs and pushes to main.
-    - name: Calculate merge-base
-      shell: bash
-      if: ${{ github.event_name == 'pull_request' }}
-      env:
-        BASE_BRANCH: ${{ github.base_ref }}
-      run: |
-        MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
-        echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV
-    - name: Restore cache (PR)
-      if: ${{ github.event_name == 'pull_request' }}
-      uses: actions/cache/restore@v3
-      with:
-        path: |
-          **/.cache
-          ~/.codeql/compile-cache
-        key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }}
-        restore-keys: |
-          codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
-          codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-
-          codeql-compile-${{ inputs.key }}-main-
-    - name: Fill cache (only branch push)
-      if: ${{ github.event_name != 'pull_request' }}
-      uses: actions/cache@v3
-      with:
-        path: |
-          **/.cache
-          ~/.codeql/compile-cache
-        key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
-        restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
-          codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
-          codeql-compile-${{ inputs.key }}-main-
-    - name: Output-compilationdir
-      id: output-compilation-dir
-      shell: bash
-      run: |
-        echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
-      env:
-        COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
-    - name: Fill compilation cache directory
-      id: fill-compilation-dir
-      uses: actions/github-script@v6
-      env: 
-        COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
-      with:
-        script: |
-          // # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-          // mkdir -p ${COMBINED_CACHE_DIR}
-          // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-          // # copy the contents of the .cache folders into the combined cache folder.
-          // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-          // # clean up the .cache folders
-          // rm -rf **/.cache/*
-
-          const fs = require("fs");
-          const path = require("path");
-          const os = require("os");
-
-          // the first argv is the cache folder to create.
-          const COMBINED_CACHE_DIR = process.env.COMBINED_CACHE_DIR;
-
-          function* walkCaches(dir) {
-            const files = fs.readdirSync(dir, { withFileTypes: true });
-            for (const file of files) {
-              if (file.isDirectory()) {
-                const filePath = path.join(dir, file.name);
-                yield* walkCaches(filePath);
-                if (file.name === ".cache") {
-                  yield filePath;
-                }
-              }
-            }
-          }
-
-          async function copyDir(src, dest) {
-            for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
-              const srcPath = path.join(src, file.name);
-              const destPath = path.join(dest, file.name);
-              if (file.isDirectory()) {
-                if (!fs.existsSync(destPath)) {
-                  fs.mkdirSync(destPath);
-                }
-                await copyDir(srcPath, destPath);
-              } else {
-                await fs.promises.copyFile(srcPath, destPath);
-              }
-            }
-          }
-
-          async function main() {
-            const cacheDirs = [...walkCaches(".")];
-
-            for (const dir of cacheDirs) {
-              console.log(`Found .cache dir at ${dir}`);
-            }
-
-            const globalCacheDir = path.join(os.homedir(), ".codeql", "compile-cache");
-            if (fs.existsSync(globalCacheDir)) {
-              console.log("Found global home dir: " + globalCacheDir);
-              cacheDirs.push(globalCacheDir);
-            }
-
-            if (cacheDirs.length === 0) {
-              console.log("No cache dirs found");
-              return;
-            }
-
-            // mkdir -p ${COMBINED_CACHE_DIR}
-            fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
-
-            // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-            await Promise.all(
-              cacheDirs.map((cacheDir) =>
-                (async function () {
-                  await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
-                  await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
-                })()
-              )
-            );
-
-            // # copy the contents of the .cache folders into the combined cache folder.
-            // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-            await Promise.all(
-              cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
-            );
-
-            // # clean up the .cache folders
-            // rm -rf **/.cache/*
-            await Promise.all(
-              cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
-            );
-          }
-          main();

.github/actions/fetch-codeql/action.yml

@@ -1,24 +1,14 @@
 name: Fetch CodeQL
 description: Fetches the latest version of CodeQL
-
-inputs:
-  channel:
-    description: 'The CodeQL channel to use'
-    required: false
-    default: 'nightly'
-
 runs:
   using: composite
   steps:
     - name: Fetch CodeQL
       shell: bash
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-        CHANNEL: ${{ inputs.channel }}
       run: |
         gh extension install github/gh-codeql
-        gh codeql set-channel "$CHANNEL"
+        gh codeql set-channel nightly
         gh codeql version
-        printf "CODEQL_FETCHED_CODEQL_PATH=" >> "${GITHUB_ENV}"
-        gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_ENV}"
         gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_PATH}"
+      env:
+        GITHUB_TOKEN: ${{ github.token }}

.github/actions/os-version/action.yml

@@ -1,32 +0,0 @@
-name: OS Version
-description: Get OS version.
-
-outputs:
-  version:
-    description: "OS version"
-    value: ${{ steps.version.outputs.version }}
-
-runs:
-  using: composite
-  steps:
-    - if: runner.os == 'Linux'
-      shell: bash
-      run: |
-        . /etc/os-release
-        echo "VERSION=${NAME} ${VERSION}" >> $GITHUB_ENV
-    - if: runner.os == 'Windows'
-      shell: powershell
-      run: |
-        $objects = systeminfo.exe /FO CSV | ConvertFrom-Csv
-        "VERSION=$($objects.'OS Name') $($objects.'OS Version')" >> $env:GITHUB_ENV
-    - if: runner.os == 'macOS'
-      shell: bash
-      run: |
-        echo "VERSION=$(sw_vers -productName) $(sw_vers -productVersion)" >> $GITHUB_ENV
-    - name: Emit OS version
-      id: version
-      shell: bash
-      run: |
-        echo "$VERSION"
-        echo "version=${VERSION}" >> $GITHUB_OUTPUT
-

.github/dependabot.yml

@@ -1,12 +1,19 @@
 version: 2
 updates:
   - package-ecosystem: "cargo"
-    directory: "ruby"
+    directory: "ruby/node-types"
     schedule:
       interval: "daily"
-
   - package-ecosystem: "cargo"
-    directory: "ql"
+    directory: "ruby/generator"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "cargo"
+    directory: "ruby/extractor"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "cargo"
+    directory: "ruby/autobuilder"
     schedule:
       interval: "daily"
 

.github/labeler.yml

@@ -51,6 +51,3 @@ documentation:
   - "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
   - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
   - "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"
-
-"ATM":
-  - javascript/ql/experimental/adaptivethreatmodeling/**/*

.github/workflows/atm-check-queries-run.yml

@@ -0,0 +1,56 @@
+name: ATM Check Queries Run
+
+env:
+  DB_PATH: test_db
+  ATM_MODEL_PACK: javascript/ql/experimental/adaptivethreatmodeling/src
+  QUERY_SUITE: codeql-suites/javascript-atm-code-scanning.qls
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/atm-check-queries-run.yml"
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+  workflow_dispatch:
+
+jobs:
+  run-atm-queries:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install CodeQL CLI
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh extensions install github/gh-codeql
+          gh codeql download
+
+      - name: Install ATM model pack
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -exu
+          
+          # Install ATM model pack
+          gh codeql pack install ${ATM_MODEL_PACK}
+
+          # Retrieve model checksum
+          model_checksum=$(gh codeql resolve extensions ${ATM_MODEL_PACK}/${QUERY_SUITE} | jq -r '.models[0].checksum')
+
+          # Trust the model so that we can use it in the ATM boosted queries
+          mkdir -p "$HOME/.config/codeql"
+          echo "--insecurely-execute-ml-model-checksums ${model_checksum}" >> "$HOME/.config/codeql/config"
+
+      - name: Create test DB
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh codeql database create ${RUNNER_TEMP}/${DB_PATH} --source-root config/atm/ --language javascript 
+
+      - name: Run ATM query suite
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh codeql database run-queries -vv -- ${RUNNER_TEMP}/${DB_PATH} ${ATM_MODEL_PACK}/${QUERY_SUITE}
+      

.github/workflows/atm-model-integration-tests.yml

@@ -0,0 +1,228 @@
+name: ATM Model Integration Tests
+
+env:
+  ATM_MODEL_PACK: javascript/ql/experimental/adaptivethreatmodeling/src
+  ATM_INTEGRATION_QUERY: javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/EndpointScoresIntegrationTest.ql
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/atm-check-queries-run.yml"
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+  workflow_dispatch:
+
+jobs:
+  run-integration-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+
+      - name: Install dependencies
+        run: |
+          pip install numpy pandas
+
+      - name: Install CodeQL CLI
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh extensions install github/gh-codeql
+          gh codeql download
+
+      - name: Install ATM model pack
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -exu
+          
+          # Install ATM model pack
+          gh codeql pack install ${ATM_MODEL_PACK}
+
+          # Retrieve model checksum
+          resolved_extensions=$(gh codeql resolve extensions ${ATM_INTEGRATION_QUERY})
+          model_checksum=$(jq -r '.models[0].checksum' <<< ${resolved_extensions})
+          model_path=$(jq -r '.models[0].path' <<< ${resolved_extensions})
+          echo "ML_MODEL_PATH=${model_path}" >> "${GITHUB_ENV}"
+
+          # Trust the model so that we can use it in the ATM boosted queries
+          mkdir -p "$HOME/.config/codeql"
+          echo "--insecurely-execute-ml-model-checksums ${model_checksum}" >> "$HOME/.config/codeql/config"
+
+      - name: Create test DB
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          DB_PATH=${RUNNER_TEMP}/test_db
+          echo "DB_PATH=${DB_PATH}" >> "${GITHUB_ENV}"
+          
+          # gh codeql database create "${DB_PATH}" --source-root config/atm/ --language javascript
+          
+          # TODO: hack
+          gh repo clone AmanSultanBaig/SignIn-SignUp-System-with-Nodejs -- --depth 1
+          gh codeql database create "${DB_PATH}" --source-root SignIn-SignUp-System-with-Nodejs/ --language javascript
+          
+          
+      - name: Run integration test query
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Run query
+          gh codeql query run \
+            --database "${DB_PATH}" \
+            --output "${RUNNER_TEMP}/integration_endpoint_scores.bqrs" \
+            ${ATM_INTEGRATION_QUERY}
+
+          # Decode results to csv
+          results_codeql="${RUNNER_TEMP}/integration_endpoint_scores.csv"
+          gh codeql bqrs decode \
+            --output "${results_codeql}" \
+            --entities "url" \
+            --format "csv" \
+            "${RUNNER_TEMP}/integration_endpoint_scores.bqrs"
+          echo "RESULTS_CODEQL=${results_codeql}" >> "${GITHUB_ENV}"
+
+      - name: Retrieve CodeQL and Python results
+        shell: python
+        run: |
+          import os
+          import numpy as np
+          import pandas as pd
+          from pathlib import Path
+
+          # CodeQL results
+          #---------------
+          df_codeql = pd.read_csv(os.environ['RESULTS_CODEQL'])
+          
+          # Replace 'URL for endpoint' column by 'url' column after stripping `file://`
+          # df_codeql['url'] = df_codeql['URL for endpoint'].map(lambda x: x[len('file://'):])  
+          
+          # TODO:hack
+          df_codeql['url'] = df_codeql['URL for endpoint'].map(lambda x: x.split('SignIn-SignUp-System-with-Nodejs/')[1] if 'SignIn-SignUp-System-with-Nodejs/' in x else x)
+          
+          df_codeql = df_codeql.drop(['URL for endpoint'], axis=1)
+          
+          # Remove results occuring in library code in`/opt/dist`
+          df_codeql = df_codeql[df_codeql['url'].map(lambda x: not x.startswith('/opt/dist'))]  
+
+          # Seralise it to csv
+          csv_codeql = f"{os.environ['RUNNER_TEMP']}/scored_endpoints_codeql.csv"
+          df_codeql.to_csv(csv_codeql)
+          with open(os.environ['GITHUB_ENV'], 'a') as f:
+            f.write(f"CSV_CODEQL={csv_codeql}\n")
+
+          # Python results
+          #---------------          
+          # Read endpoints locations and scores files packaged with ML model
+          model_path = Path(os.environ['ML_MODEL_PATH'])
+          df_model_locations = pd.read_csv(model_path.joinpath('model_checks', 'endpoint_locations.csv'))
+          df_model_scores = pd.read_csv(model_path.joinpath('model_checks', 'endpoint_scores.csv'))
+          
+          # Make the `url` column the same as `df_codeql` e.g. /opt/src/auth/authMiddleware.js:1:21:1:34
+          df_model_locations['url'] = \
+              df_model_locations['absolutePath'] + \
+              ':' + \
+              df_model_locations['startLine'].astype(str) + \
+              ':' + \
+              df_model_locations['startColumn'].astype(str) + \
+              ':' + \
+              df_model_locations['endLine'].astype(str) + \
+              ':' + \
+              df_model_locations['endColumn'].astype(str)
+          
+          # TODO: hack
+          df_model_locations['url'] = df_model_locations['url'].map(lambda x: x.split('/opt/src/')[1] if '/opt/src/' in x else x)
+          
+          # Merge locations and scores
+          df_model = df_model_scores.merge(df_model_locations)
+          df_model = df_model.drop(
+              [
+                  'entityName',
+                  'startLine',
+                  'startColumn',
+                  'endLine',
+                  'endColumn',
+                  'absolutePath',
+              ], axis=1
+          )
+          df_model = df_model.rename(columns={'scoreIndex': 'encodedEndpointType', 'scoreValue': 'score'})
+
+          # Seralise it to csv
+          csv_model = f"{os.environ['RUNNER_TEMP']}/scored_endpoints_pyton.csv"
+          df_model.to_csv(csv_model)
+          with open(os.environ['GITHUB_ENV'], 'a') as f:
+            f.write(f"CSV_MODEL={csv_model}\n")
+            
+      - name: Check endpoints locations
+        shell: python
+        run: |
+          # All the results in the `model_checks` directory packaged with the model appear when running 
+          # `EndpointScoresIntegrationTest.ql` against the model check DB
+
+          # NOTE: why do we have different number of results?
+          #
+          # model_check results:
+          # The model checks datataset created by the pipeline contains endpoints labelled as positive or negative
+          # examples. These endpoints are scored by the model training script once the model has been trained.
+          #
+          # codeql results:
+          # These are produced by the EndpointScoresIntegrationTest.ql query which scores all endpoints that are
+          # DataFlow::CallNode. This *happens to* encompass pretty much all possible endpoints and so will also contain
+          # the endpoints used by model_checks, along with several more.
+          #
+          # The reason we have more codeql endpoints than model_checks endpoints is thus because we do not have an easy
+          # way of selecting the same endpoints and rely on a hack to get a set of endpoints that will encompass the
+          # endpoints that we actually care about (but this is not a theoretical guarantee, more of a heuristic).
+
+          import os
+          import numpy as np
+          import pandas as pd
+          
+          # Read in data
+          df_codeql = pd.read_csv(os.environ['CSV_CODEQL'])
+          df_model = pd.read_csv(os.environ['CSV_MODEL'])
+          df_all_inner = df_codeql.merge(df_model, on=['url', 'encodedEndpointType'], suffixes=('_codeql', '_model'))
+          df_all_outer = df_codeql.merge(df_model, on=['url', 'encodedEndpointType'], suffixes=('_codeql', '_model'), how='outer', indicator=True)
+
+          # Display number of endpoints
+          print(f'Number of codeql results (i.e. from running `{os.environ["ATM_INTEGRATION_QUERY"].split("/")[-1]}`): {df_codeql.shape[0]}')
+          print(f'Number of model checks results (i.e. shipped with model): {df_model.shape[0]}')
+          print(f'Number of overlapping results: {df_all_inner.shape[0]}')
+
+          # Check all model checks results are also in codeql results
+          df_all_diff = df_all_outer[df_all_outer._merge != 'both']
+          print(f'Number of results in `model_check` but not in `codeql` (should be zero): {len(df_all_diff[df_all_diff["_merge"] == "right_only"])}')
+          print(f'Number of results in `codeql` but not in `model_checks` (expect non-zero): {len(df_all_diff[df_all_diff["_merge"] == "left_only"])}')
+          if len(df_all_diff[df_all_diff["_merge"] == "right_only"]):
+              print(f'Missing results when running `EndpointScoresIntegrationTest.ql` that are in `model_check`: {df_all_diff[df_all_diff["_merge"] == "right_only"].to_string()}')
+          assert df_model.shape[0] == df_all_inner.shape[0], f'There are missing results from `model_check` when running `EndpointScoresIntegrationTest.ql`'
+
+      - name: Check endpoints scores
+        shell: python
+        run: |
+          # The scores produced by the CodeQL query `EndpointScoresIntegrationTest.ql` should match those packaged
+          # with the model.
+          
+          import os
+          import numpy as np
+          import pandas as pd
+          
+          # Read in data
+          df_codeql = pd.read_csv(os.environ['CSV_CODEQL'])
+          df_model = pd.read_csv(os.environ['CSV_MODEL'])
+          df_all_inner = df_codeql.merge(df_model, on=['url', 'encodedEndpointType'], suffixes=('_codeql', '_model'))
+
+          # Assert the codeql and model checks scores are almost identical
+          rtol=1e-04
+          np.testing.assert_allclose(
+              df_all_inner['score_codeql'],
+              df_all_inner['score_model'],
+              rtol=rtol,
+          ), f'There are non-matching scores'
+          print(f'The scores of the {df_all_inner.shape[0]} overlapping endpoints match, using rtol={rtol}.')
+
+
+      

.github/workflows/check-change-note.yml

@@ -8,7 +8,6 @@ on:
       - "*/ql/src/**/*.qll"
       - "*/ql/lib/**/*.ql"
       - "*/ql/lib/**/*.qll"
-      - "*/ql/lib/**/*.yml"
       - "!**/experimental/**"
       - "!ql/**"
       - "!swift/**"
@@ -27,9 +26,3 @@ jobs:
         run: |
           gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' |
           grep true -c
-      - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md' or 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text.
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$"))' |
-          grep true -c

.github/workflows/check-qldoc.yml

@@ -26,8 +26,9 @@ jobs:
         shell: bash
         run: |
           EXIT_CODE=0
+          # TODO: remove the swift exception from the regex when we fix generated QLdoc
           # TODO: remove the shared exception from the regex when coverage of qlpacks without dbschemes is supported
-          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!(shared))[a-z]*/ql/lib' || true; } | sort -u)"
+          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!(swift|shared))[a-z]*/ql/lib' || true; } | sort -u)"
           for pack_dir in ${changed_lib_packs}; do
             lang="${pack_dir%/ql/lib}"
             codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"

.github/workflows/check-query-ids.yml

@@ -1,21 +0,0 @@
-name: Check query IDs
-
-on:
-  pull_request:
-    paths:
-      - "**/src/**/*.ql"
-      - misc/scripts/check-query-ids.py
-      - .github/workflows/check-query-ids.yml
-    branches:
-      - main
-      - "rc/*"
-  workflow_dispatch:
-
-jobs:
-  check:
-    name: Check query IDs
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check for duplicate query IDs
-        run: python3 misc/scripts/check-query-ids.py

.github/workflows/close-stale.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/stale@v8
+    - uses: actions/stale@v6
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Comment or remove the `Stale` label in order to avoid having this issue closed in 7 days.'

.github/workflows/codeql-analysis.yml

@@ -28,9 +28,9 @@ jobs:
 
     steps:
     - name: Setup dotnet
-      uses: actions/setup-dotnet@v3
+      uses: actions/setup-dotnet@v2
       with:
-        dotnet-version: 7.0.102
+        dotnet-version: 6.0.202
 
     - name: Checkout repository
       uses: actions/checkout@v3

.github/workflows/compile-queries.yml

@@ -1,37 +0,0 @@
-name: "Compile all queries using the latest stable CodeQL CLI"
-
-on:
-  push:
-    branches:  # makes sure the cache gets populated - running on the branches people tend to merge into.
-      - main
-      - "rc/*"
-      - "codeql-cli-*"
-  pull_request:
-
-jobs:
-  compile-queries:
-    runs-on: ubuntu-latest-xl
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup CodeQL
-        uses: ./.github/actions/fetch-codeql
-        with:
-          channel: 'release'
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: all-queries
-      - name: check formatting
-        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 -n 3000 -P 10 codeql query format -q --check-only
-      - name: compile queries - check-only
-        # run with --check-only if running in a PR (github.sha != main)
-        if : ${{ github.event_name == 'pull_request' }}
-        shell: bash
-        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-      - name: compile queries - full
-        # do full compile if running on main - this populates the cache
-        if : ${{ github.event_name != 'pull_request' }}
-        shell: bash
-        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/csharp-qltest.yml

@@ -1,86 +0,0 @@
-name: "C#: Run QL Tests"
-
-on:
-  push:
-    paths:
-      - "csharp/**"
-      - "shared/**"
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "csharp/**"
-      - "shared/**"
-      - .github/workflows/csharp-qltest.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
-
-defaults:
-  run:
-    working-directory: csharp
-
-jobs:
-  qlupgrade:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - name: Check DB upgrade scripts
-        run: |
-          echo >empty.trap
-          codeql dataset import -S ql/lib/upgrades/initial/semmlecode.csharp.dbscheme testdb empty.trap
-          codeql dataset upgrade testdb --additional-packs ql/lib
-          diff -q testdb/semmlecode.csharp.dbscheme ql/lib/semmlecode.csharp.dbscheme
-      - name: Check DB downgrade scripts
-        run: |
-          echo >empty.trap
-          rm -rf testdb; codeql dataset import -S ql/lib/semmlecode.csharp.dbscheme testdb empty.trap
-          codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
-           --dbscheme=ql/lib/semmlecode.csharp.dbscheme --target-dbscheme=downgrades/initial/semmlecode.csharp.dbscheme |
-           xargs codeql execute upgrades testdb
-          diff -q testdb/semmlecode.csharp.dbscheme downgrades/initial/semmlecode.csharp.dbscheme
-  qltest:
-    runs-on: ubuntu-latest-xl
-    strategy:
-      fail-fast: false
-      matrix:
-        slice: ["1/2", "2/2"]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - uses: ./csharp/actions/create-extractor-pack
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with:
-          key: csharp-qltest-${{ matrix.slice }}
-      - name: Run QL tests
-        run: |
-          CODEQL_PATH=$(gh codeql version --format=json | jq -r .unpackedLocation)
-          # The legacy ASP extractor is not in this repo, so take the one from the nightly build
-          mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
-          # Safe guard against using the bundled extractor
-          rm -rf "$CODEQL_PATH/csharp"
-          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-  unit-tests:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup dotnet
-        uses: actions/setup-dotnet@v3
-        with:
-          dotnet-version: 7.0.102
-      - name: Extractor unit tests
-        run: |
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/extractor/Semmle.Util.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/extractor/Semmle.Extraction.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"

.github/workflows/fast-forward.yml

@@ -1,50 +0,0 @@
-# Fast-forwards the branch specified in BRANCH_NAME
-# to the github.ref/sha that this workflow is run on.
-# Used as part of the release process, to ensure
-# external query writers can always access a branch of github/codeql
-# that is compatible with the latest stable release.
-name: Fast-forward tracking branch for selected CodeQL version
-on:
-  workflow_dispatch:
-
-jobs:
-  fast-forward:
-    name: Fast-forward tracking branch for selected CodeQL version
-    runs-on: ubuntu-latest
-    if: github.repository == 'github/codeql'
-    permissions:
-      contents: write
-    env:
-      BRANCH_NAME: 'lgtm.com'
-    steps:
-      - name: Validate chosen branch
-        if: ${{ !startsWith(github.ref_name, 'codeql-cli-') }}
-        shell: bash
-        run: |
-          echo "::error ::The $BRANCH_NAME tracking branch should only be fast-forwarded to the tip of a codeql-cli-* branch, got $GITHUB_REF_NAME instead."
-          exit 1
-
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Git config
-        shell: bash
-        run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-
-      - name: Fetch
-        shell: bash
-        run: |
-          set -x
-          echo "Fetching $BRANCH_NAME"
-          # Explicitly unshallow and fetch to ensure the remote ref is available.
-          git fetch --unshallow origin "$BRANCH_NAME"
-          git checkout -b "$BRANCH_NAME" "origin/$BRANCH_NAME"
-
-      - name: Fast-forward
-        shell: bash
-        run: |
-          echo "Fast-forwarding $BRANCH_NAME to ${GITHUB_REF}@${GITHUB_SHA}"
-          git merge --ff-only "$GITHUB_SHA"
-          git push origin "$BRANCH_NAME"

.github/workflows/go-tests-other-os.yml

@@ -1,80 +0,0 @@
-name: "Go: Run Tests - Other OS"
-on:
-  pull_request:
-    paths:
-      - "go/**"
-      - "!go/ql/**" # don't run other-os if only ql/ files changed
-      - .github/workflows/go-tests-other-os.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-jobs:
-  test-mac:
-    name: Test MacOS
-    runs-on: macos-latest
-    steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.20.0
-        id: go
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up CodeQL CLI
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Enable problem matchers in repository
-        shell: bash
-        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-      - name: Build
-        run: |
-          cd go
-          make
-
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with:
-          key: go-qltest
-      - name: Test
-        run: |
-          cd go
-          make test cache="${{ steps.query-cache.outputs.cache-dir }}"
-
-  test-win:
-    name: Test Windows
-    runs-on: windows-latest-xl
-    steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.20.0
-        id: go
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up CodeQL CLI
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Enable problem matchers in repository
-        shell: bash
-        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-      - name: Build
-        run: |
-          cd go
-          make
-
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with:
-          key: go-qltest
-
-      - name: Test
-        run: |
-          cd go
-          make test cache="${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/go-tests.yml

@@ -1,29 +1,20 @@
 name: "Go: Run Tests"
 on:
-  push:
-    paths:
-      - "go/**"
-      - .github/workflows/go-tests.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
   pull_request:
     paths:
       - "go/**"
       - .github/workflows/go-tests.yml
-      - .github/actions/**
+      - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
 jobs:
   test-linux:
     name: Test Linux (Ubuntu)
-    runs-on: ubuntu-latest-xl
+    runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v4
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
         with:
-          go-version: 1.20.0
+          go-version: 1.19
         id: go
 
       - name: Check out code
@@ -41,7 +32,7 @@ jobs:
           cd go
           make
 
-      - name: Check that all Go code is autoformatted
+      - name: Check that all QL and Go code is autoformatted
         run: |
           cd go
           make check-formatting
@@ -57,13 +48,67 @@ jobs:
           name: qhelp-markdown
           path: go/qhelp-out/**/*.md
 
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
+      - name: Test
+        run: |
+          cd go
+          make test
+
+  test-mac:
+    name: Test MacOS
+    runs-on: macos-latest
+    steps:
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
         with:
-          key: go-qltest
+          go-version: 1.19
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
 
       - name: Test
         run: |
           cd go
-          make test cache="${{ steps.query-cache.outputs.cache-dir }}"
+          make test
+
+  test-win:
+    name: Test Windows
+    runs-on: windows-2019
+    steps:
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Test
+        run: |
+          cd go
+          make test

.github/workflows/js-ml-tests.yml

@@ -23,9 +23,22 @@ defaults:
     working-directory: javascript/ql/experimental/adaptivethreatmodeling
 
 jobs:
-  qltest:
-    name: Test QL
-    runs-on: ubuntu-latest-xl
+  qlformat:
+    name: Check QL formatting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Check QL formatting
+        run: |
+          find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
+            xargs -0 codeql query format --check-only
+
+  qlcompile:
+    name: Check QL compilation
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
@@ -33,33 +46,36 @@ jobs:
 
       - name: Install pack dependencies
         run: |
-          for pack in modelbuilding src test; do
+          for pack in modelbuilding src; do
             codeql pack install --mode verify -- "${pack}"
           done
-      
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: js-ml-test
 
       - name: Check QL compilation
         run: |
           codeql query compile \
             --check-only \
-            --ram 50000 \
+            --ram 5120 \
             --additional-packs "${{ github.workspace }}" \
             --threads=0 \
-            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
             -- \
             lib modelbuilding src
 
+  qltest:
+    name: Run QL tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Install pack dependencies
+        run: codeql pack install -- test
+
       - name: Run QL tests
         run: |
           codeql test run \
             --threads=0 \
-            --ram 50000 \
+            --ram 5120 \
             --additional-packs "${{ github.workspace }}" \
-            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
             -- \
-            test
+            test

.github/workflows/mad_modelDiff.yml

@@ -11,7 +11,7 @@ on:
     branches:
       - main
     paths:
-      - "java/ql/src/utils/modelgenerator/**/*.*"
+      - "java/ql/src/utils/model-generator/**/*.*"
       - ".github/workflows/mad_modelDiff.yml"
 
 permissions:
@@ -40,12 +40,12 @@ jobs:
       - name: Download database
         env:
           SLUG: ${{ matrix.slug }}
-          GH_TOKEN: ${{ github.token }}
         run: |
           set -x
           mkdir lib-dbs
           SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
-          gh api -H "Accept: application/zip" "/repos/${SLUG}/code-scanning/codeql/databases/java" > "$SHORTNAME.zip"
+          projectId=`curl -s https://lgtm.com/api/v1.0/projects/g/${SLUG} | jq .id`
+          curl -L "https://lgtm.com/api/v1.0/snapshots/$projectId/java" -o "$SHORTNAME.zip"
           unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
           mkdir "lib-dbs/$SHORTNAME/"
           mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/"
@@ -61,8 +61,8 @@ jobs:
             DATABASE=$2
             cd codeql-$QL_VARIANT
             SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE ${SHORTNAME}.temp.model.yml
-            mv java/ql/lib/ext/generated/${SHORTNAME}.temp.model.yml $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.model.yml
+            python java/ql/src/utils/model-generator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $MODELS/${SHORTNAME}.qll
+            mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
             cd ..
           }
 
@@ -85,21 +85,19 @@ jobs:
           set -x
           MODELS=`pwd`/tmp-models
           ls -1 tmp-models/
-          for m in $MODELS/*_main.model.yml ; do
+          for m in $MODELS/*_main.qll ; do
             t="${m/main/"pr"}"
             basename=`basename $m`
-            name="diff_${basename/_main.model.yml/""}"
+            name="diff_${basename/_main.qll/""}"
             (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
           done
       - uses: actions/upload-artifact@v3
         with:
           name: models
-          path: tmp-models/*.model.yml
+          path: tmp-models/*.qll
           retention-days: 20
       - uses: actions/upload-artifact@v3
         with:
           name: diffs
           path: tmp-models/*.html
-          # An html file is only produced if the generated models differ.
-          if-no-files-found: ignore
           retention-days: 20

.github/workflows/mad_regenerate-models.yml

@@ -50,10 +50,10 @@ jobs:
           SLUG: ${{ matrix.slug }}
         run: |
           SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
-          java/ql/src/utils/modelgenerator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
+          java/ql/src/utils/model-generator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
       - name: Stage changes
         run: |
-          find java -name "*.model.yml" -print0 | xargs -0 git add
+          find java -name "*.qll" -print0 | xargs -0 git add
           git status
           git diff --cached > models.patch
       - uses: actions/upload-artifact@v3

.github/workflows/ql-for-ql-build.yml

@@ -5,6 +5,13 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+    paths:
+      - "ql/**"
+      - "**.qll"
+      - "**.ql"
+      - "**.dbscheme"
+      - "**/qlpack.yml"
+      - ".github/workflows/ql-for-ql-build.yml"
 
 env:
   CARGO_TERM_COLOR: always
@@ -15,60 +22,137 @@ jobs:
     steps:
       ### Build the queries ###
       - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: javascript # does not matter
-      - uses: ./.github/actions/os-version
-        id: os_version
+      - name: Get CodeQL version
+        id: get-codeql-version
+        run: |
+          echo "::set-output name=version::$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)"
+        shell: bash
+        env:
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+      - name: Cache entire pack
+        id: cache-pack
+        uses: actions/cache@v3
+        with:
+          path: ${{ runner.temp }}/pack
+          key: ${{ runner.os }}-pack-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
+      - name: Cache queries
+        if: steps.cache-pack.outputs.cache-hit != 'true'
+        id: cache-queries
+        uses: actions/cache@v3
+        with:
+          path: ${{ runner.temp }}/queries
+          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
+      - name: Build query pack
+        if: steps.cache-queries.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: |
+          cd ql/ql/src
+          "${CODEQL}" pack create -j 16
+          mv .codeql/pack/codeql/ql/0.0.0 ${{ runner.temp }}/queries
+        env:
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+      - name: Move cache queries to pack
+        if: steps.cache-pack.outputs.cache-hit != 'true'
+        run: |
+          cp -r ${{ runner.temp }}/queries ${{ runner.temp }}/pack
+        env:
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+
       ### Build the extractor ###
       - name: Cache entire extractor
+        if: steps.cache-pack.outputs.cache-hit != 'true'
         id: cache-extractor
         uses: actions/cache@v3
         with:
           path: |
-            ql/extractor-pack/
-            ql/target/release/buramu
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
+            ql/target/release/ql-autobuilder
+            ql/target/release/ql-autobuilder.exe
+            ql/target/release/ql-extractor
+            ql/target/release/ql-extractor.exe
+          key: ${{ runner.os }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
       - name: Cache cargo
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         uses: actions/cache@v3
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
+          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
+      - name: Check formatting
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: cd ql; cargo fmt --all -- --check
+      - name: Build
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: cd ql; cargo build --verbose
+      - name: Run tests
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: cd ql; cargo test --verbose
       - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; ./scripts/create-extractor-pack.sh       
-        env:
-          GH_TOKEN: ${{ github.token }}   
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: run-ql-for-ql
-      - name: Make database and analyze
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: cd ql; cargo build --release
+      - name: Generate dbscheme
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
+
+      ### Package the queries and extractor ###
+      - name: Package pack
+        if: steps.cache-pack.outputs.cache-hit != 'true'
         run: |
-          ./ql/target/release/buramu | tee deprecated.blame # Add a blame file for the extractor to parse.
-          ${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB}
-          ${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls  --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-        env: 
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-          DB: ${{ runner.temp }}/DB
-          LGTM_INDEX_FILTERS: |
-            exclude:ql/ql/test
-            exclude:*/ql/lib/upgrades/
-            exclude:java/ql/integration-tests
-      - name: Upload sarif to code-scanning
-        uses: github/codeql-action/upload-sarif@v2
+          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats ${PACK}/
+          mkdir -p ${PACK}/tools/linux64
+          cp ql/target/release/ql-autobuilder  ${PACK}/tools/linux64/autobuilder
+          cp ql/target/release/ql-extractor ${PACK}/tools/linux64/extractor
+          chmod +x ${PACK}/tools/linux64/autobuilder
+          chmod +x ${PACK}/tools/linux64/extractor
+        env:
+          PACK: ${{ runner.temp }}/pack
+
+      ### Run the analysis ###
+      - name: Hack codeql-action options
+        run: |
+          JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .resolve.languages=["--search-path", $pack] | .database.init=["--search-path", $pack]')
+          echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
+        env:
+          PACK: ${{ runner.temp }}/pack
+
+      - name: Create CodeQL config file
+        run: |
+          echo "paths-ignore:" >> ${CONF}
+          echo "  - ql/ql/test" >> ${CONF}
+          echo "  - \"*/ql/lib/upgrades/\"" >> ${CONF}
+          echo "disable-default-queries: true" >> ${CONF}
+          echo "queries:" >> ${CONF}
+          echo "  - uses: ./ql/ql/src/codeql-suites/ql-code-scanning.qls" >> ${CONF}
+          echo "Config file: "
+          cat ${CONF}
+        env:
+          CONF: ./ql-for-ql-config.yml
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
-          sarif_file: ql-for-ql.sarif
-          category: ql-for-ql
+          languages: ql
+          db-location: ${{ runner.temp }}/db
+          config-file: ./ql-for-ql-config.yml
+      - name: Move pack cache
+        run: |
+          cp -r ${PACK}/.cache ql/ql/src/.cache
+        env:
+          PACK: ${{ runner.temp }}/pack
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        with:
+          category: "ql-for-ql"
+      - name: Copy sarif file to CWD
+        run: cp ../results/ql.sarif ./ql-for-ql.sarif
+      - name: Fixup the $scema in sarif  # Until https://github.com/microsoft/sarif-vscode-extension/pull/436/ is part in a stable release
+        run: |
+          sed -i 's/\$schema.*/\$schema": "https:\/\/raw.githubusercontent.com\/oasis-tcs\/sarif-spec\/master\/Schemata\/sarif-schema-2.1.0",/' ql-for-ql.sarif
       - name: Sarif as artifact
         uses: actions/upload-artifact@v3
         with:
@@ -83,4 +167,4 @@ jobs:
         with:
           name: ql-for-ql-langs
           path: split-sarif
-          retention-days: 1
+          retention-days: 1

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -25,18 +25,16 @@ jobs:
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: javascript # does not matter
-      - uses: ./.github/actions/os-version
-        id: os_version
       - uses: actions/cache@v3
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
+          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Build Extractor
         run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./scripts/create-extractor-pack.sh
         env:

.github/workflows/ql-for-ql-tests.yml

@@ -6,13 +6,11 @@ on:
     paths:
       - "ql/**"
       - codeql-workspace.yml
-      - .github/workflows/ql-for-ql-tests.yml
   pull_request:
     branches: [main]
     paths:
       - "ql/**"
       - codeql-workspace.yml
-      - .github/workflows/ql-for-ql-tests.yml
 
 env:
   CARGO_TERM_COLOR: always
@@ -24,86 +22,33 @@ jobs:
       - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: javascript # does not matter
-      - uses: ./.github/actions/os-version
-        id: os_version
       - uses: actions/cache@v3
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/rust-toolchain.toml', 'ql/**/Cargo.lock') }}
-      - name: Check formatting
-        run: cd ql; cargo fmt --all -- --check
+          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Build extractor
         run: |
           cd ql;
           codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
           env "PATH=$PATH:$codeqlpath" ./scripts/create-extractor-pack.sh
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: ql-for-ql-tests
       - name: Run QL tests
         run: |
-          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
+          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-
-  other-os: 
-    strategy:
-      matrix:
-        os: [macos-latest, windows-latest]
-    needs: [qltest]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install GNU tar 
-        if: runner.os == 'macOS'
+      - name: Check QL formatting
         run: |
-          brew install gnu-tar
-          echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@v2
-        with:
-          languages: javascript # does not matter
-      - uses: ./.github/actions/os-version
-        id: os_version
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/rust-toolchain.toml', 'ql/**/Cargo.lock') }}
-      - name: Build extractor
-        if: runner.os != 'Windows'
-        run: |
-          cd ql;
-          codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
-          env "PATH=$PATH:$codeqlpath" ./scripts/create-extractor-pack.sh
-      - name: Build extractor (Windows)
-        if: runner.os == 'Windows'
-        shell: pwsh
-        run: |
-          cd ql;
-          $Env:PATH += ";$(dirname ${{ steps.find-codeql.outputs.codeql-path }})"
-          pwsh ./scripts/create-extractor-pack.ps1
-      - name: Run a single QL tests - Unix
-        if: runner.os != 'Windows'
-        run: |
-          "${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
+          find ql/ql/src "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Run a single QL tests - Windows
-        if: runner.os == 'Windows'
-        shell: pwsh
+      - name: Check QL compilation
         run: |
-          $Env:PATH += ";$(dirname ${{ steps.find-codeql.outputs.codeql-path }})"
-          codeql test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
-      
+          "${CODEQL}" query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}/ql/extractor-pack" "ql/ql/src" "ql/ql/examples"
+        env:
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}

.github/workflows/ruby-build.yml

@@ -48,55 +48,24 @@ jobs:
         run: |
           brew install gnu-tar
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
-      - name: Install cargo-cross
-        if: runner.os == 'Linux'
-        run: cargo install cross --version 0.2.5
-      - uses: ./.github/actions/os-version
-        id: os_version
-      - name: Cache entire extractor
-        uses: actions/cache@v3
-        id: cache-extractor
-        with:
-          path: |
-            ruby/extractor/target/release/autobuilder
-            ruby/extractor/target/release/autobuilder.exe
-            ruby/extractor/target/release/extractor
-            ruby/extractor/target/release/extractor.exe
-            ruby/extractor/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}--${{ hashFiles('ruby/extractor/**/*.rs') }}
       - uses: actions/cache@v3
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
             ruby/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-rust-cargo-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/**/Cargo.lock') }}
+          key: ${{ runner.os }}-ruby-rust-cargo-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}
       - name: Check formatting
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd extractor && cargo fmt --all -- --check
+        run: cargo fmt --all -- --check
       - name: Build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd extractor && cargo build --verbose
+        run: cargo build --verbose
       - name: Run tests
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd extractor && cargo test --verbose
-      # On linux, build the extractor via cross in a centos7 container.
-      # This ensures we don't depend on glibc > 2.17.
-      - name: Release build (linux)
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os == 'Linux'
-        run: |
-          cd extractor
-          cross build --release
-          mv target/x86_64-unknown-linux-gnu/release/extractor target/release/
-          mv target/x86_64-unknown-linux-gnu/release/autobuilder target/release/
-          mv target/x86_64-unknown-linux-gnu/release/generator target/release/
-      - name: Release build (windows and macos)
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os != 'Linux'
-        run: cd extractor && cargo build --release
+        run: cargo test --verbose
+      - name: Release build
+        run: cargo build --release
       - name: Generate dbscheme
-        if: ${{ matrix.os == 'ubuntu-latest' && steps.cache-extractor.outputs.cache-hit != 'true'}}
-        run: extractor/target/release/generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: target/release/ruby-generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v3
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
@@ -111,40 +80,33 @@ jobs:
         with:
           name: extractor-${{ matrix.os }}
           path: |
-            ruby/extractor/target/release/autobuilder
-            ruby/extractor/target/release/autobuilder.exe
-            ruby/extractor/target/release/extractor
-            ruby/extractor/target/release/extractor.exe
+            ruby/target/release/ruby-autobuilder
+            ruby/target/release/ruby-autobuilder.exe
+            ruby/target/release/ruby-extractor
+            ruby/target/release/ruby-extractor.exe
           retention-days: 1
   compile-queries:
-    runs-on: ubuntu-latest-xl
+    runs-on: ubuntu-latest
+    env:
+      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
     steps:
       - uses: actions/checkout@v3
       - name: Fetch CodeQL
         uses: ./.github/actions/fetch-codeql
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: ruby-build
       - name: Build Query Pack
         run: |
-          PACKS=${{ runner.temp }}/query-packs
-          rm -rf $PACKS
-          codeql pack create ../misc/suite-helpers --output "$PACKS"
-          codeql pack create ../shared/regex --output "$PACKS"
-          codeql pack create ../shared/ssa --output "$PACKS"
-          codeql pack create ../shared/tutorial --output "$PACKS"
-          codeql pack create ql/lib --output "$PACKS"
-          codeql pack create -j0 ql/src --output "$PACKS" --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-          PACK_FOLDER=$(readlink -f "$PACKS"/codeql/ruby-queries/*)
+          codeql pack create ../shared/ssa --output target/packs
+          codeql pack create ../misc/suite-helpers --output target/packs
+          codeql pack create ql/lib --output target/packs
+          codeql pack create ql/src --output target/packs
+          PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
           codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
           (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
       - uses: actions/upload-artifact@v3
         with:
           name: codeql-ruby-queries
           path: |
-            ${{ runner.temp }}/query-packs/*
+            ruby/target/packs/*
           retention-days: 1
 
   package:
@@ -172,12 +134,12 @@ jobs:
           mkdir -p ruby
           cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
           mkdir -p ruby/tools/{linux64,osx64,win64}
-          cp linux64/autobuilder ruby/tools/linux64/autobuilder
-          cp osx64/autobuilder ruby/tools/osx64/autobuilder
-          cp win64/autobuilder.exe ruby/tools/win64/autobuilder.exe
-          cp linux64/extractor ruby/tools/linux64/extractor
-          cp osx64/extractor ruby/tools/osx64/extractor
-          cp win64/extractor.exe ruby/tools/win64/extractor.exe
+          cp linux64/ruby-autobuilder ruby/tools/linux64/autobuilder
+          cp osx64/ruby-autobuilder ruby/tools/osx64/autobuilder
+          cp win64/ruby-autobuilder.exe ruby/tools/win64/autobuilder.exe
+          cp linux64/ruby-extractor ruby/tools/linux64/extractor
+          cp osx64/ruby-extractor ruby/tools/osx64/extractor
+          cp win64/ruby-extractor.exe ruby/tools/win64/extractor.exe
           chmod +x ruby/tools/{linux64,osx64}/{autobuilder,extractor}
           zip -rq codeql-ruby.zip ruby
       - uses: actions/upload-artifact@v3
@@ -219,6 +181,11 @@ jobs:
       - name: Fetch CodeQL
         uses: ./.github/actions/fetch-codeql
 
+      - uses: actions/checkout@v3
+        with:
+          repository: Shopify/example-ruby-app
+          ref: 67a0decc5eb550f3a9228eda53925c3afd40dfe9
+
       - name: Download Ruby bundle
         uses: actions/download-artifact@v3
         with:
@@ -227,67 +194,27 @@ jobs:
       - name: Unzip Ruby bundle
         shell: bash
         run: unzip -q -d "${{ runner.temp }}/ruby-bundle" "${{ runner.temp }}/codeql-ruby-bundle.zip"
-
+      - name: Prepare test files
+        shell: bash
+        run: |
+          echo "import codeql.ruby.AST select count(File f)" > "test.ql"
+          echo "| 4 |" > "test.expected"
+          echo 'name: sample-tests
+          version: 0.0.0
+          dependencies:
+            codeql/ruby-all: "*"
+          extractor: ruby
+          tests: .
+          ' > qlpack.yml
       - name: Run QL test
         shell: bash
         run: |
-          codeql test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" ruby/ql/test/library-tests/ast/constants/
+          codeql test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
       - name: Create database
         shell: bash
         run: |
-          codeql database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root ruby/ql/test/library-tests/ast/constants/ ../database
+          codeql database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
       - name: Analyze database
         shell: bash
         run: |
           codeql database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls
-
-  # This is a copy of the 'test' job that runs in a centos7 container.
-  # This tests that the extractor works correctly on systems with an old glibc.
-  test-centos7:
-    defaults:
-      run:
-        working-directory: ${{ github.workspace }}
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-latest
-    container:
-      image: centos:centos7
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    needs: [package]
-    steps:
-      - name: Install gh cli
-        run: |
-          yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
-          # fetch-codeql requires unzip and jq
-          # jq is available in epel-release (https://docs.fedoraproject.org/en-US/epel/)
-          yum install -y gh unzip epel-release
-          yum install -y jq
-      - uses: actions/checkout@v3
-      - name: Fetch CodeQL
-        uses: ./.github/actions/fetch-codeql
-
-      # Due to a bug in Actions, we can't use runner.temp in the run blocks here.
-      # https://github.com/actions/runner/issues/2185
-
-      - name: Download Ruby bundle
-        uses: actions/download-artifact@v3
-        with:
-          name: codeql-ruby-bundle
-          path: ${{ runner.temp }}
-      - name: Unzip Ruby bundle
-        shell: bash
-        run: unzip -q -d "$RUNNER_TEMP"/ruby-bundle "$RUNNER_TEMP"/codeql-ruby-bundle.zip
-
-      - name: Run QL test
-        shell: bash
-        run: |
-          codeql test run --search-path "$RUNNER_TEMP"/ruby-bundle --additional-packs "$RUNNER_TEMP"/ruby-bundle ruby/ql/test/library-tests/ast/constants/
-      - name: Create database
-        shell: bash
-        run: |
-          codeql database create --search-path "$RUNNER_TEMP"/ruby-bundle --language ruby --source-root ruby/ql/test/library-tests/ast/constants/ ../database
-      - name: Analyze database
-        shell: bash
-        run: |
-          codeql database analyze --search-path "$RUNNER_TEMP"/ruby-bundle --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/ruby-qltest.yml

@@ -4,8 +4,7 @@ on:
   push:
     paths:
       - "ruby/**"
-      - "shared/**"
-      - .github/workflows/ruby-build.yml
+      - .github/workflows/ruby-qltest.yml
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
     branches:
@@ -29,6 +28,23 @@ defaults:
     working-directory: ruby
 
 jobs:
+  qlformat:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - name: Check QL formatting
+        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
+  qlcompile:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - name: Check QL compilation
+        run: |
+          codeql query compile --check-only --threads=0 --ram 5000 --warnings=error "ql/src" "ql/examples"
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
   qlupgrade:
     runs-on: ubuntu-latest
     steps:
@@ -49,20 +65,17 @@ jobs:
            xargs codeql execute upgrades testdb
           diff -q testdb/ruby.dbscheme downgrades/initial/ruby.dbscheme
   qltest:
-    runs-on: ubuntu-latest-xl
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
+      matrix:
+        slice: ["1/2", "2/2"]
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - uses: ./ruby/actions/create-extractor-pack
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: ruby-qltest
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 5000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/swift-autobuilder.yml

@@ -0,0 +1,27 @@
+name: "Swift: Build and test Xcode autobuilder"
+
+on:
+  pull_request:
+    paths:
+      - "swift/xcode-autobuilder/**"
+      - "misc/bazel/**"
+      - "*.bazel*"
+      - .github/workflows/swift-autobuilder.yml
+    branches:
+      - main
+
+jobs:
+  autobuilder:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version-file: 'swift/.python-version'
+      - name: Build the Xcode autobuilder
+        run: |
+          bazel build //swift/xcode-autobuilder
+      - name: Test the Xcode autobuilder
+        run: |
+          bazel test //swift/xcode-autobuilder/tests

.github/workflows/swift-codegen.yml

@@ -0,0 +1,44 @@
+name: "Swift: Check code generation"
+
+on:
+  pull_request:
+    paths:
+      - "swift/**"
+      - "misc/bazel/**"
+      - "*.bazel*"
+      - .github/workflows/swift-codegen.yml
+      - .github/actions/fetch-codeql/action.yml
+    branches:
+      - main
+defaults:
+  run:
+    working-directory: swift
+
+jobs:
+  codegen:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version-file: 'swift/.python-version'
+      - uses: pre-commit/action@v3.0.0
+        name: Check that python code is properly formatted
+        with:
+          extra_args: autopep8 --all-files
+      - name: Run unit tests
+        run: |
+          bazel test //swift/codegen/test --test_output=errors
+      - uses: pre-commit/action@v3.0.0
+        name: Check that QL generated code was checked in
+        with:
+          extra_args: swift-codegen --all-files
+      - name: Generate C++ files
+        run: |
+          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-cpp-files
+      - uses: actions/upload-artifact@v3
+        with:
+          name: swift-generated-cpp-files
+          path: swift-generated-cpp-files/**

.github/workflows/swift-integration-tests.yml

@@ -0,0 +1,47 @@
+name: "Swift: Run Integration Tests"
+
+on:
+  pull_request:
+    paths:
+      - "swift/**"
+      - "misc/bazel/**"
+      - "*.bazel*"
+      - .github/workflows/swift-integration-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+defaults:
+  run:
+    working-directory: swift
+
+jobs:
+  integration-tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-20.04
+#          - macos-latest  TODO
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version-file: 'swift/.python-version'
+      - name: Build Swift extractor
+        run: |
+          bazel run //swift:create-extractor-pack
+      - name: Get Swift version
+        id: get_swift_version
+        run: |
+          VERSION=$(bazel run //swift/extractor -- --version | sed -ne 's/.*version \(\S*\).*/\1/p')
+          echo "::set-output name=version::$VERSION"
+      - uses: swift-actions/setup-swift@v1
+        with:
+          swift-version: "${{steps.get_swift_version.outputs.version}}"
+      - name: Run integration tests
+        run: |
+          python integration-tests/runner.py

.github/workflows/swift-qltest.yml

@@ -0,0 +1,57 @@
+name: "Swift: Run QL Tests"
+
+on:
+  pull_request:
+    paths:
+      - "swift/**"
+      - "misc/bazel/**"
+      - "*.bazel*"
+      - .github/workflows/swift-qltest.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+defaults:
+  run:
+    working-directory: swift
+
+jobs:
+  qlformat:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - name: Check QL formatting
+        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
+  qltest-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version-file: 'swift/.python-version'
+      - name: Test qltest.sh
+        run: |
+          bazel test //swift/tools/test/qltest
+  qltest:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-20.04, macos-latest ]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version-file: 'swift/.python-version'
+      - name: Build Swift extractor
+        run: |
+          bazel run //swift:create-extractor-pack
+      - name: Run QL tests
+        run: |
+          codeql test run --threads=0 --ram 5000 --search-path "${{ github.workspace }}/swift/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition ql/test
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/swift.yml

@@ -1,105 +0,0 @@
-name: "Swift"
-
-on:
-  pull_request:
-    paths:
-      - "swift/**"
-      - "misc/bazel/**"
-      - "misc/codegen/**"
-      - "*.bazel*"
-      - .github/workflows/swift.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-      - .pre-commit-config.yaml
-      - "!**/*.md"
-      - "!**/*.qhelp"
-    branches:
-      - main
-      - rc/*
-  push:
-    paths:
-      - "swift/**"
-      - "misc/bazel/**"
-      - "misc/codegen/**"
-      - "*.bazel*"
-      - .github/workflows/swift.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-      - "!**/*.md"
-      - "!**/*.qhelp"
-    branches:
-      - main
-      - rc/*
-
-jobs:
-  # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks
-  # without waiting for the macOS build
-  build-and-test-macos:
-    runs-on: macos-12-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/build-and-test
-  build-and-test-linux:
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/build-and-test
-  qltests-linux:
-    needs: build-and-test-linux
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/run-ql-tests
-  qltests-macos:
-    if : ${{ github.event_name == 'pull_request' }}
-    needs: build-and-test-macos
-    runs-on: macos-12-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/run-ql-tests
-  integration-tests-linux:
-    needs: build-and-test-linux
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/run-integration-tests
-  integration-tests-macos:
-    if : ${{ github.event_name == 'pull_request' }}
-    needs: build-and-test-macos
-    runs-on: macos-12-xl
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/run-integration-tests
-  codegen:
-    if : ${{ github.event_name == 'pull_request' }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: bazelbuild/setup-bazelisk@v2
-      - uses: actions/setup-python@v4
-        with:
-          python-version-file: 'swift/.python-version'
-      - uses: pre-commit/action@v3.0.0
-        name: Check that python code is properly formatted
-        with:
-          extra_args: autopep8 --all-files
-      - uses: ./.github/actions/fetch-codeql
-      - uses: pre-commit/action@v3.0.0
-        name: Check that QL generated code was checked in
-        with:
-          extra_args: swift-codegen --all-files
-      - name: Generate C++ files
-        run: |
-          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/generated-cpp-files
-      - uses: actions/upload-artifact@v3
-        with:
-          name: swift-generated-cpp-files
-          path: generated-cpp-files/**
-  database-upgrade-scripts:
-    if : ${{ github.event_name == 'pull_request' }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - uses: ./swift/actions/database-upgrade-scripts

.gitignore

@@ -27,6 +27,8 @@
 # It's useful (though not required) to be able to unpack codeql in the ql checkout itself
 /codeql/
 
+csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
+
 # Avoid committing cached package components
 .codeql
 

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
     rev: v1.6.0
     hooks:
       - id: autopep8
-        files: ^misc/codegen/.*\.py
+        files: ^swift/codegen/.*\.py
 
   - repo: local
     hooks:
@@ -31,7 +31,7 @@ repos:
 
       - id: sync-files
         name: Fix files required to be identical
-        files: \.(qll?|qhelp|swift)$|^config/identical-files\.json$
+        files: \.(qll?|qhelp|swift)$
         language: system
         entry: python3 config/sync-files.py --latest
         pass_filenames: false
@@ -44,7 +44,7 @@ repos:
 
       - id: swift-codegen
         name: Run Swift checked in code generation
-        files: ^swift/(schema.py$|codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements)|ql/\.generated.list)
+        files: ^swift/(codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements))
         language: system
         entry: bazel run //swift/codegen -- --quiet
         pass_filenames: false
@@ -53,5 +53,5 @@ repos:
         name: Run Swift code generation unit tests
         files: ^swift/codegen/.*\.py$
         language: system
-        entry: bazel test //misc/codegen/test
+        entry: bazel test //swift/codegen/test
         pass_filenames: false

.vscode/settings.json

@@ -1,5 +1,3 @@
 {
-  "omnisharp.autoStart": false,
-  "cmake.sourceDirectory": "${workspaceFolder}/swift",
-  "cmake.buildDirectory": "${workspaceFolder}/bazel-cmake-build"
+    "omnisharp.autoStart": false
 }

CODEOWNERS

@@ -5,14 +5,20 @@
 /javascript/ @github/codeql-javascript
 /python/ @github/codeql-python
 /ruby/ @github/codeql-ruby
-/swift/ @github/codeql-swift
-/misc/codegen/ @github/codeql-swift
+/swift/ @github/codeql-c
 /java/kotlin-extractor/ @github/codeql-kotlin
 /java/kotlin-explorer/ @github/codeql-kotlin
 
 # ML-powered queries
 /javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
 
+# Notify members of codeql-go about PRs to the shared data-flow library files
+/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl.qll @github/codeql-java @github/codeql-go
+/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl2.qll @github/codeql-java @github/codeql-go
+/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll @github/codeql-java @github/codeql-go
+/java/ql/src/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll @github/codeql-java @github/codeql-go
+/java/ql/src/semmle/code/java/dataflow/internal/tainttracking2/TaintTrackingImpl.qll @github/codeql-java @github/codeql-go
+
 # CodeQL tools and associated docs
 /docs/codeql/codeql-cli/ @github/codeql-cli-reviewers
 /docs/codeql/codeql-for-visual-studio-code/ @github/codeql-vscode-reviewers
@@ -34,9 +40,8 @@ WORKSPACE.bazel @github/codeql-ci-reviewers
 
 # Workflows
 /.github/workflows/ @github/codeql-ci-reviewers
-/.github/workflows/atm-* @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/go-* @github/codeql-go
 /.github/workflows/js-ml-tests.yml @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
 /.github/workflows/ruby-* @github/codeql-ruby
-/.github/workflows/swift.yml @github/codeql-swift
+/.github/workflows/swift-* @github/codeql-c

CONTRIBUTING.md

@@ -25,7 +25,6 @@ If you have an idea for a query that you would like to share with other CodeQL u
 
     Each language-specific directory contains further subdirectories that group queries based on their `@tags` or purpose.
     - Experimental queries and libraries are stored in the `experimental` subdirectory within each language-specific directory in the [CodeQL repository](https://github.com/github/codeql). For example, experimental Java queries and libraries are stored in `java/ql/src/experimental` and any corresponding tests in `java/ql/test/experimental`.
-    - Experimental queries need to include `experimental` in their `@tags`
     - The structure of an `experimental` subdirectory mirrors the structure of its parent directory.
     - Select or create an appropriate directory in `experimental` based on the existing directory structure of `experimental` or its parent directory.
 

README.md

@@ -10,8 +10,6 @@ There is [extensive documentation](https://codeql.github.com/docs/) on getting s
 
 We welcome contributions to our standard library and standard checks. Do you have an idea for a new check, or how to improve an existing query? Then please go ahead and open a pull request! Before you do, though, please take the time to read our [contributing guidelines](CONTRIBUTING.md). You can also consult our [style guides](https://github.com/github/codeql/tree/main/docs) to learn how to format your code for consistency and clarity, how to write query metadata, and how to write query help documentation for your query.
 
-For information on contributing to CodeQL documentation, see the "[contributing guide](docs/codeql/CONTRIBUTING.md)" for docs.
-
 ## License
 
 The code in this repository is licensed under the [MIT License](LICENSE) by [GitHub](https://github.com).

codeql-workspace.yml

@@ -17,7 +17,6 @@ provide:
   # - "javascript/ql/experimental/adaptivethreatmodeling/model/qlpack.yml"
   - "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml"
   - "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/test/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/lib/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/src/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/test/qlpack.yml"
@@ -25,8 +24,7 @@ provide:
   - "misc/suite-helpers/qlpack.yml"
   - "ruby/extractor-pack/codeql-extractor.yml"
   - "swift/extractor-pack/codeql-extractor.yml"
-  - "swift/integration-tests/qlpack.yml"
-  - "ql/extractor-pack/codeql-extractor.yml"
+  - "ql/extractor-pack/codeql-extractor.ym"
 
 versionPolicies:
   default:

config/identical-files.json

@@ -1,93 +1,67 @@
 {
-  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift": [
-    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlow.qll",
-    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlow.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlow.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlow.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlow.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlow.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlow.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlow.qll"
-  ],
-  "DataFlowImpl Java/C++/C#/Go/Python/Ruby/Swift": [
+  "DataFlow Java/C++/C#/Python": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll",
-    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"
-  ],
-  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift Legacy Configuration": [
-    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl1.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl2.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl3.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl4.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl5.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl6.qll",
-    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl1.qll",
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForSerializability.qll",
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForOnActivityResult.qll",
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl4.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplLocal.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl1.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl1.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl2.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplForContentDataFlow.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl1.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl2.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImplForStringsNewReplacer.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl1.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplForRegExp.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl1.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForRegExp.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForHttpClientLibraries.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForPathname.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl1.qll"
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"
   ],
-  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift Common": [
+  "DataFlow Java/C++/C#/Python Common": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImplCommon.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll"
   ],
-  "TaintTracking Java/C++/C#/Go/Python/Ruby/Swift": [
-    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTracking.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTracking.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking1/TaintTracking.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/tainttracking1/TaintTracking.qll",
-    "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTracking.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking1/TaintTracking.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTracking.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/tainttracking1/TaintTracking.qll"
-  ],
-  "TaintTracking Legacy Configuration Java/C++/C#/Go/Python/Ruby/Swift": [
+  "TaintTracking::Configuration Java/C++/C#/Python": [
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking4/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking5/TaintTrackingImpl.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
@@ -98,19 +72,19 @@
     "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
   ],
-  "DataFlow Java/C++/C#/Python/Ruby/Swift Consistency checks": [
+  "DataFlow Java/C++/C#/Python Consistency checks": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplConsistency.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplConsistency.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplConsistency.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplConsistency.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplConsistency.qll"
   ],
-  "DataFlow Java/C#/Go/Ruby/Python/Swift Flow Summaries": [
+  "DataFlow Java/C#/Ruby/Python/Swift Flow Summaries": [
     "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/FlowSummaryImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/FlowSummaryImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/FlowSummaryImpl.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/FlowSummaryImpl.qll"
@@ -120,12 +94,8 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll"
   ],
   "Model as Data Generation Java/C# - CaptureModels": [
-    "java/ql/src/utils/modelgenerator/internal/CaptureModels.qll",
-    "csharp/ql/src/utils/modelgenerator/internal/CaptureModels.qll"
-  ],
-  "Model as Data Generation Java/C# - CaptureModelsPrinting": [
-    "java/ql/src/utils/modelgenerator/internal/CaptureModelsPrinting.qll",
-    "csharp/ql/src/utils/modelgenerator/internal/CaptureModelsPrinting.qll"
+    "java/ql/src/utils/model-generator/internal/CaptureModels.qll",
+    "csharp/ql/src/utils/model-generator/internal/CaptureModels.qll"
   ],
   "Sign Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/Sign.qll",
@@ -283,11 +253,6 @@
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/IRBlockImports.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/IRBlockImports.qll"
   ],
-  "C++ IR IRConsistencyImports": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/IRConsistencyImports.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/IRConsistencyImports.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/IRConsistencyImports.qll"
-  ],
   "C++ IR IRFunctionImports": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/IRFunctionImports.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/IRFunctionImports.qll",
@@ -431,6 +396,16 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/ControlFlowReachability.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/ControlFlowReachability.qll"
   ],
+  "Inline Test Expectations": [
+    "cpp/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "csharp/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "java/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "python/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "ruby/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "ql/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "go/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "swift/ql/test/TestUtilities/InlineExpectationsTest.qll"
+  ],
   "C++ ExternalAPIs": [
     "cpp/ql/src/Security/CWE/CWE-020/ExternalAPIs.qll",
     "cpp/ql/src/Security/CWE/CWE-020/ir/ExternalAPIs.qll"
@@ -489,10 +464,6 @@
     "javascript/ql/src/Comments/CommentedOutCodeReferences.inc.qhelp",
     "python/ql/src/Lexical/CommentedOutCodeReferences.inc.qhelp"
   ],
-  "ThreadResourceAbuse qhelp": [
-    "java/ql/src/experimental/Security/CWE/CWE-400/LocalThreadResourceAbuse.qhelp",
-    "java/ql/src/experimental/Security/CWE/CWE-400/ThreadResourceAbuse.qhelp"
-  ],
   "IDE Contextual Queries": [
     "cpp/ql/lib/IDEContextual.qll",
     "csharp/ql/lib/IDEContextual.qll",
@@ -515,6 +486,40 @@
     "python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll",
     "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll"
   ],
+  "ReDoS Util Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/NfaUtils.qll",
+    "python/ql/lib/semmle/python/security/regexp/NfaUtils.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/NfaUtils.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/NfaUtils.qll"
+  ],
+  "ReDoS Exponential Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/ExponentialBackTracking.qll",
+    "python/ql/lib/semmle/python/security/regexp/ExponentialBackTracking.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/ExponentialBackTracking.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/ExponentialBackTracking.qll"
+  ],
+  "ReDoS Polynomial Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/SuperlinearBackTracking.qll",
+    "python/ql/lib/semmle/python/security/regexp/SuperlinearBackTracking.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/SuperlinearBackTracking.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/SuperlinearBackTracking.qll"
+  ],
+  "RegexpMatching Python/JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/RegexpMatching.qll",
+    "python/ql/lib/semmle/python/security/regexp/RegexpMatching.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/RegexpMatching.qll"
+  ],
+  "BadTagFilterQuery Python/JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll",
+    "python/ql/lib/semmle/python/security/BadTagFilterQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/BadTagFilterQuery.qll"
+  ],
+  "OverlyLargeRange Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/OverlyLargeRangeQuery.qll",
+    "python/ql/lib/semmle/python/security/OverlyLargeRangeQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/OverlyLargeRangeQuery.qll",
+    "java/ql/lib/semmle/code/java/security/OverlyLargeRangeQuery.qll"
+  ],
   "CFG": [
     "csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll",
     "ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll",
@@ -524,9 +529,16 @@
     "python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll",
     "ruby/ql/lib/codeql/ruby/typetracking/TypeTracker.qll"
   ],
+  "CodeQL Tutorial": [
+    "cpp/ql/lib/tutorial.qll",
+    "csharp/ql/lib/tutorial.qll",
+    "java/ql/lib/tutorial.qll",
+    "javascript/ql/lib/tutorial.qll",
+    "python/ql/lib/tutorial.qll",
+    "ruby/ql/lib/tutorial.qll"
+  ],
   "AccessPathSyntax": [
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/AccessPathSyntax.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/AccessPathSyntax.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/AccessPathSyntax.qll",
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/AccessPathSyntax.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll",
@@ -542,16 +554,16 @@
     "ruby/ql/lib/codeql/ruby/internal/ConceptsShared.qll",
     "javascript/ql/lib/semmle/javascript/internal/ConceptsShared.qll"
   ],
+  "Hostname Regexp queries": [
+    "javascript/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
+    "python/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
+    "ruby/ql/src/queries/security/cwe-020/HostnameRegexpShared.qll"
+  ],
   "ApiGraphModels": [
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll",
     "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll",
     "python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll"
   ],
-  "ApiGraphModelsExtensions": [
-    "javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModelsExtensions.qll",
-    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModelsExtensions.qll",
-    "python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsExtensions.qll"
-  ],
   "TaintedFormatStringQuery Ruby/JS": [
     "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringQuery.qll",
     "ruby/ql/lib/codeql/ruby/security/TaintedFormatStringQuery.qll"
@@ -588,16 +600,8 @@
     "swift/ql/test/extractor-tests/patterns/patterns.swift",
     "swift/ql/test/library-tests/ast/patterns.swift"
   ],
-  "Swift control flow test file": [
-    "swift/ql/test/library-tests/controlflow/graph/cfg.swift",
-    "swift/ql/test/library-tests/ast/cfg.swift"
-  ],
   "IncompleteMultiCharacterSanitization JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/IncompleteMultiCharacterSanitizationQuery.qll",
     "ruby/ql/lib/codeql/ruby/security/IncompleteMultiCharacterSanitizationQuery.qll"
-  ],
-  "EncryptionKeySizes Python/Java": [
-    "python/ql/lib/semmle/python/security/internal/EncryptionKeySizes.qll",
-    "java/ql/lib/semmle/code/java/security/internal/EncryptionKeySizes.qll"
   ]
 }

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -1,6 +1,5 @@
 using Xunit;
 using Semmle.Autobuild.Shared;
-using Semmle.Util;
 using System.Collections.Generic;
 using System;
 using System.Linq;
@@ -76,15 +75,6 @@ namespace Semmle.Autobuild.Cpp.Tests
             throw new ArgumentException("Missing RunProcess " + pattern);
         }
 
-        int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
-        {
-            var ret = (this as IBuildActions).RunProcess(cmd, args, workingDirectory, env, out var stdout);
-
-            stdout.ForEach(line => onOutput(line));
-
-            return ret;
-        }
-
         public IList<string> DirectoryDeleteIn = new List<string>();
 
         void IBuildActions.DirectoryDelete(string dir, bool recursive)
@@ -141,14 +131,6 @@ namespace Semmle.Autobuild.Cpp.Tests
 
         bool IBuildActions.IsWindows() => IsWindows;
 
-        public bool IsMacOs { get; set; }
-
-        bool IBuildActions.IsMacOs() => IsMacOs;
-
-        public bool IsArm { get; set; }
-
-        bool IBuildActions.IsArm() => IsArm;
-
         string IBuildActions.PathCombine(params string[] parts)
         {
             return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));
@@ -194,15 +176,6 @@ namespace Semmle.Autobuild.Cpp.Tests
             if (!DownloadFiles.Contains((address, fileName)))
                 throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");
         }
-
-        public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new TestDiagnosticWriter();
-    }
-
-    internal class TestDiagnosticWriter : IDiagnosticsWriter
-    {
-        public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
-
-        public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
     }
 
     /// <summary>
@@ -262,7 +235,6 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
-            Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
             Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
@@ -285,11 +257,11 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.GetCurrentDirectory = cwd;
             Actions.IsWindows = isWindows;
 
-            var options = new CppAutobuildOptions(Actions);
+            var options = new AutobuildOptions(Actions, Language.Cpp);
             return new CppAutobuilder(Actions, options);
         }
 
-        void TestAutobuilderScript(CppAutobuilder autobuilder, int expectedOutput, int commandsRun)
+        void TestAutobuilderScript(Autobuilder autobuilder, int expectedOutput, int commandsRun)
         {
             Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(Actions, StartCallback, EndCallback));
 
@@ -327,7 +299,7 @@ namespace Semmle.Autobuild.Cpp.Tests
         {
             Actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test.sln -DisableParallelProcessing"] = 1;
             Actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test.sln -DisableParallelProcessing"] = 0;
-            Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && msbuild C:\Project\test.sln /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"""] = 0;
+            Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0;
             Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0;

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
     <Nullable>enable</Nullable>
@@ -11,12 +11,11 @@
   <ItemGroup>
     <PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
     <PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
-    <PackageReference Include="xunit" Version="2.4.2" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
   </ItemGroup>
 
   <ItemGroup>

cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs

@@ -1,28 +1,10 @@
 using Semmle.Autobuild.Shared;
-using Semmle.Util;
 
 namespace Semmle.Autobuild.Cpp
 {
-    /// <summary>
-    /// Encapsulates C++ build options.
-    /// </summary>
-    public class CppAutobuildOptions : AutobuildOptionsShared
+    public class CppAutobuilder : Autobuilder
     {
-        public override Language Language => Language.Cpp;
-
-
-        /// <summary>
-        /// Reads options from environment variables.
-        /// Throws ArgumentOutOfRangeException for invalid arguments.
-        /// </summary>
-        public CppAutobuildOptions(IBuildActions actions) : base(actions)
-        {
-        }
-    }
-
-    public class CppAutobuilder : Autobuilder<CppAutobuildOptions>
-    {
-        public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options, new DiagnosticClassifier()) { }
+        public CppAutobuilder(IBuildActions actions, AutobuildOptions options) : base(actions, options) { }
 
         public override BuildScript GetBuildScript()
         {

cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs

@@ -11,14 +11,14 @@ namespace Semmle.Autobuild.Cpp
             try
             {
                 var actions = SystemBuildActions.Instance;
-                var options = new CppAutobuildOptions(actions);
+                var options = new AutobuildOptions(actions, Language.Cpp);
                 try
                 {
                     Console.WriteLine("CodeQL C++ autobuilder");
                     var builder = new CppAutobuilder(actions, options);
                     return builder.AttemptBuild();
                 }
-                catch (InvalidEnvironmentException ex)
+                catch(InvalidEnvironmentException ex)
                 {
                     Console.WriteLine("The environment is invalid: {0}", ex.Message);
                 }

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <AssemblyName>Semmle.Autobuild.Cpp</AssemblyName>
     <RootNamespace>Semmle.Autobuild.Cpp</RootNamespace>
     <ApplicationIcon />
@@ -17,7 +17,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Build" Version="17.3.2" />
+    <PackageReference Include="Microsoft.Build" Version="16.11.0" />
   </ItemGroup>
 
   <ItemGroup>

cpp/downgrades/19887dbd33327fb07d54251786e0cb2578539775/upgrade.properties

@@ -1,4 +0,0 @@
-description: Revert support for repeated initializers, which are allowed in C with designated initializers.
-compatibility: full
-aggregate_field_init.rel: reorder aggregate_field_init.rel (int aggregate, int initializer, int field, int position) aggregate initializer field
-aggregate_array_init.rel: reorder aggregate_array_init.rel (int aggregate, int initializer, int element_index, int position) aggregate initializer element_index

cpp/downgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/exprs.ql

@@ -13,5 +13,5 @@ predicate isExprWithNewBuiltin(Expr expr) {
 from Expr expr, int kind, int kind_new, Location location
 where
   exprs(expr, kind, location) and
-  if isExprWithNewBuiltin(expr) then kind_new = 1 else kind_new = kind
+  if isExprWithNewBuiltin(expr) then kind_new = 0 else kind_new = kind
 select expr, kind_new, location

cpp/downgrades/73af5058c6899dcdb05754c27ca966aeb3a68c94/exprs.ql

@@ -9,5 +9,5 @@ class Location extends @location_expr {
 from Expr expr, int kind, int kind_new, Location location
 where
   exprs(expr, kind, location) and
-  if expr instanceof @blockassignexpr then kind_new = 1 else kind_new = kind
+  if expr instanceof @blockassignexpr then kind_new = 0 else kind_new = kind
 select expr, kind_new, location

cpp/downgrades/a5bb28ed29f73855d64cc5f939cef977fa8fd19a/builtintypes.ql

@@ -1,11 +0,0 @@
-class BuiltinType extends @builtintype {
-  string toString() { none() }
-}
-
-from BuiltinType type, string name, int kind, int kind_new, int size, int sign, int alignment
-where
-  builtintypes(type, name, kind, size, sign, alignment) and
-  if type instanceof @float16 or type instanceof @complex_float16
-  then kind_new = 2
-  else kind_new = kind
-select type, name, kind_new, size, sign, alignment

cpp/downgrades/a5bb28ed29f73855d64cc5f939cef977fa8fd19a/upgrade.properties

@@ -1,3 +0,0 @@
-description: Introduce (_Complex) _Float16 type
-compatibility: backwards
-builtintypes.rel: run builtintypes.qlo

cpp/downgrades/ba86bebea4c7a8235c2fa0e220391fbd4446a087/upgrade.properties

@@ -1,2 +0,0 @@
-description: Uncomment case splits in dbscheme
-compatibility: full

cpp/ql/examples/qlpack.yml

@@ -3,4 +3,4 @@ groups:
   - cpp
   - examples
 dependencies:
-  codeql/cpp-all: ${workspace}
+  codeql/cpp-all: "*"

cpp/ql/lib/CHANGELOG.md

@@ -1,131 +1,3 @@
-## 0.7.0
-
-### Breaking Changes
-
-* The internal `SsaConsistency` module has been moved from `SSAConstruction` to `SSAConsitency`, and the deprecated `SSAConsistency` module has been removed.
-
-### Deprecated APIs
-
-* The single-parameter predicates `ArrayOrVectorAggregateLiteral.getElementExpr` and `ClassAggregateLiteral.getFieldExpr` have been deprecated in favor of `ArrayOrVectorAggregateLiteral.getAnElementExpr` and `ClassAggregateLiteral.getAFieldExpr`.
-* The recently introduced new data flow and taint tracking APIs have had a
-  number of module and predicate renamings. The old APIs remain in place for
-  now.
-* The `SslContextCallAbstractConfig`, `SslContextCallConfig`, `SslContextCallBannedProtocolConfig`, `SslContextCallTls12ProtocolConfig`, `SslContextCallTls13ProtocolConfig`, `SslContextCallTlsProtocolConfig`, `SslContextFlowsToSetOptionConfig`, `SslOptionConfig` dataflow configurations from `BoostorgAsio` have been deprecated. Please use `SslContextCallConfigSig`, `SslContextCallGlobal`, `SslContextCallFlow`, `SslContextCallBannedProtocolFlow`, `SslContextCallTls12ProtocolFlow`, `SslContextCallTls13ProtocolFlow`, `SslContextCallTlsProtocolFlow`, `SslContextFlowsToSetOptionFlow`.
-
-### New Features
-
-* Added overridable predicates `getSizeExpr` and `getSizeMult` to the `BufferAccess` class (`semmle.code.cpp.security.BufferAccess.qll`). This makes it possible to model a larger class of buffer reads and writes using the library.
-
-### Minor Analysis Improvements
-
-* The `BufferAccess` library (`semmle.code.cpp.security.BufferAccess`) no longer matches buffer accesses inside unevaluated contexts (such as inside `sizeof` or `decltype` expressions). As a result, queries using this library may see fewer false positives.
-
-### Bug Fixes
-
-* Fixed some accidental predicate visibility in the backwards-compatible wrapper for data flow configurations. In particular `DataFlow::hasFlowPath`, `DataFlow::hasFlow`, `DataFlow::hasFlowTo`, and `DataFlow::hasFlowToExpr` were accidentally exposed in a single version.
-
-## 0.6.1
-
-No user-facing changes.
-
-## 0.6.0
-
-### Breaking Changes
-
-* The `semmle.code.cpp.commons.Buffer` and `semmle.code.cpp.commons.NullTermination` libraries no longer expose `semmle.code.cpp.dataflow.DataFlow`. Please import `semmle.code.cpp.dataflow.DataFlow` directly.
-
-### Deprecated APIs
-
-* The `WriteConfig` taint tracking configuration has been deprecated. Please use `WriteFlow`.
-
-### New Features
-
-* Added support for merging two `PathGraph`s via disjoint union to allow results from multiple data flow computations in a single `path-problem` query.
-
-### Major Analysis Improvements
-
-* A new C/C++ dataflow library (`semmle.code.cpp.dataflow.new.DataFlow`) has been added.
-  The new library behaves much more like the dataflow library of other CodeQL supported
-  languages by following use-use dataflow paths instead of def-use dataflow paths.
-  The new library also better supports dataflow through indirections, and new predicates
-  such as `Node::asIndirectExpr` have been added to facilitate working with indirections.
-
-  The `semmle.code.cpp.ir.dataflow.DataFlow` library is now identical to the new
-  `semmle.code.cpp.dataflow.new.DataFlow` library.
-* The main data flow and taint tracking APIs have been changed. The old APIs
-  remain in place for now and translate to the new through a
-  backwards-compatible wrapper. If multiple configurations are in scope
-  simultaneously, then this may affect results slightly. The new API is quite
-  similar to the old, but makes use of a configuration module instead of a
-  configuration class.
-
-### Minor Analysis Improvements
-
-* Deleted the deprecated `hasGeneratedCopyConstructor` and `hasGeneratedCopyAssignmentOperator` predicates from the `Folder` class.
-* Deleted the deprecated `getPath` and `getFolder` predicates from the `XmlFile` class.
-* Deleted the deprecated `getMustlockFunction`, `getTrylockFunction`, `getLockFunction`, and `getUnlockFunction` predicates from the `MutexType` class.
-* Deleted the deprecated `getPosInBasicBlock` predicate from the `SubBasicBlock` class.
-* Deleted the deprecated `getExpr` predicate from the `PointerDereferenceExpr` class.
-* Deleted the deprecated `getUseInstruction` and `getDefinitionInstruction` predicates from the `Operand` class.
-* Deleted the deprecated `isInParameter`, `isInParameterPointer`, and `isInQualifier` predicates from the `FunctionInput` class.
-* Deleted the deprecated `isOutParameterPointer`, `isOutQualifier`, `isOutReturnValue`, and `isOutReturnPointer` predicate from the `FunctionOutput` class.
-* Deleted the deprecated 3-argument `isGuardPhi` predicate from the `RangeSsaDefinition` class.
-
-## 0.5.4
-
-No user-facing changes.
-
-## 0.5.3
-
-No user-facing changes.
-
-## 0.5.2
-
-No user-facing changes.
-
-## 0.5.1
-
-No user-facing changes.
-
-## 0.5.0
-
-### Breaking Changes
-
-The predicates in the `MustFlow::Configuration` class used by the `MustFlow` library (`semmle.code.cpp.ir.dataflow.MustFlow`) have changed to be defined directly in terms of the C++ IR instead of IR dataflow nodes.
-
-### Deprecated APIs
-
-* Deprecated `semmle.code.cpp.ir.dataflow.DefaultTaintTracking`. Use `semmle.code.cpp.ir.dataflow.TaintTracking`.
-* Deprecated `semmle.code.cpp.security.TaintTrackingImpl`. Use `semmle.code.cpp.ir.dataflow.TaintTracking`.
-* Deprecated `semmle.code.cpp.valuenumbering.GlobalValueNumberingImpl`. Use `semmle.code.cpp.valuenumbering.GlobalValueNumbering`, which exposes the same API.
-
-### Minor Analysis Improvements
-
-* The `ArgvSource` flow source now uses the second parameter of `main` as its source instead of the uses of this parameter.
-* The `ArgvSource` flow source has been generalized to handle cases where the argument vector of `main` is not named `argv`.
-* The `getaddrinfo` function is now recognized as a flow source.
-* The `secure_getenv` and `_wgetenv` functions are now recognized as local flow sources.
-* The `scanf` and `fscanf` functions and their variants are now recognized as flow sources.
-* Deleted the deprecated `getName` and `getShortName` predicates from the `Folder` class.
-
-## 0.4.6
-
-No user-facing changes.
-
-## 0.4.5
-
-No user-facing changes.
-
-## 0.4.4
-
-No user-facing changes.
-
-## 0.4.3
-
-### Minor Analysis Improvements
-
-* Fixed bugs in the `FormatLiteral` class that were causing `getMaxConvertedLength` and related predicates to return no results when the format literal was `%e`, `%f` or `%g` and an explicit precision was specified.
-
 ## 0.4.2
 
 No user-facing changes.

cpp/ql/lib/change-notes/2022-10-22-format-literal.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Fixed bugs in the `FormatLiteral` class that were causing `getMaxConvertedLength` and related predicates to return no results when the format literal was `%e`, `%f` or `%g` and an explicit precision was specified.

cpp/ql/lib/change-notes/released/0.4.3.md

@@ -1,5 +0,0 @@
-## 0.4.3
-
-### Minor Analysis Improvements
-
-* Fixed bugs in the `FormatLiteral` class that were causing `getMaxConvertedLength` and related predicates to return no results when the format literal was `%e`, `%f` or `%g` and an explicit precision was specified.

cpp/ql/lib/change-notes/released/0.4.4.md

@@ -1,3 +0,0 @@
-## 0.4.4
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.4.5.md

@@ -1,3 +0,0 @@
-## 0.4.5
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.4.6.md

@@ -1,3 +0,0 @@
-## 0.4.6
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.5.0.md

@@ -1,20 +0,0 @@
-## 0.5.0
-
-### Breaking Changes
-
-The predicates in the `MustFlow::Configuration` class used by the `MustFlow` library (`semmle.code.cpp.ir.dataflow.MustFlow`) have changed to be defined directly in terms of the C++ IR instead of IR dataflow nodes.
-
-### Deprecated APIs
-
-* Deprecated `semmle.code.cpp.ir.dataflow.DefaultTaintTracking`. Use `semmle.code.cpp.ir.dataflow.TaintTracking`.
-* Deprecated `semmle.code.cpp.security.TaintTrackingImpl`. Use `semmle.code.cpp.ir.dataflow.TaintTracking`.
-* Deprecated `semmle.code.cpp.valuenumbering.GlobalValueNumberingImpl`. Use `semmle.code.cpp.valuenumbering.GlobalValueNumbering`, which exposes the same API.
-
-### Minor Analysis Improvements
-
-* The `ArgvSource` flow source now uses the second parameter of `main` as its source instead of the uses of this parameter.
-* The `ArgvSource` flow source has been generalized to handle cases where the argument vector of `main` is not named `argv`.
-* The `getaddrinfo` function is now recognized as a flow source.
-* The `secure_getenv` and `_wgetenv` functions are now recognized as local flow sources.
-* The `scanf` and `fscanf` functions and their variants are now recognized as flow sources.
-* Deleted the deprecated `getName` and `getShortName` predicates from the `Folder` class.

cpp/ql/lib/change-notes/released/0.5.1.md

@@ -1,3 +0,0 @@
-## 0.5.1
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.5.2.md

@@ -1,3 +0,0 @@
-## 0.5.2
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.5.3.md

@@ -1,3 +0,0 @@
-## 0.5.3
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.5.4.md

@@ -1,3 +0,0 @@
-## 0.5.4
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.6.0.md

@@ -1,42 +0,0 @@
-## 0.6.0
-
-### Breaking Changes
-
-* The `semmle.code.cpp.commons.Buffer` and `semmle.code.cpp.commons.NullTermination` libraries no longer expose `semmle.code.cpp.dataflow.DataFlow`. Please import `semmle.code.cpp.dataflow.DataFlow` directly.
-
-### Deprecated APIs
-
-* The `WriteConfig` taint tracking configuration has been deprecated. Please use `WriteFlow`.
-
-### New Features
-
-* Added support for merging two `PathGraph`s via disjoint union to allow results from multiple data flow computations in a single `path-problem` query.
-
-### Major Analysis Improvements
-
-* A new C/C++ dataflow library (`semmle.code.cpp.dataflow.new.DataFlow`) has been added.
-  The new library behaves much more like the dataflow library of other CodeQL supported
-  languages by following use-use dataflow paths instead of def-use dataflow paths.
-  The new library also better supports dataflow through indirections, and new predicates
-  such as `Node::asIndirectExpr` have been added to facilitate working with indirections.
-
-  The `semmle.code.cpp.ir.dataflow.DataFlow` library is now identical to the new
-  `semmle.code.cpp.dataflow.new.DataFlow` library.
-* The main data flow and taint tracking APIs have been changed. The old APIs
-  remain in place for now and translate to the new through a
-  backwards-compatible wrapper. If multiple configurations are in scope
-  simultaneously, then this may affect results slightly. The new API is quite
-  similar to the old, but makes use of a configuration module instead of a
-  configuration class.
-
-### Minor Analysis Improvements
-
-* Deleted the deprecated `hasGeneratedCopyConstructor` and `hasGeneratedCopyAssignmentOperator` predicates from the `Folder` class.
-* Deleted the deprecated `getPath` and `getFolder` predicates from the `XmlFile` class.
-* Deleted the deprecated `getMustlockFunction`, `getTrylockFunction`, `getLockFunction`, and `getUnlockFunction` predicates from the `MutexType` class.
-* Deleted the deprecated `getPosInBasicBlock` predicate from the `SubBasicBlock` class.
-* Deleted the deprecated `getExpr` predicate from the `PointerDereferenceExpr` class.
-* Deleted the deprecated `getUseInstruction` and `getDefinitionInstruction` predicates from the `Operand` class.
-* Deleted the deprecated `isInParameter`, `isInParameterPointer`, and `isInQualifier` predicates from the `FunctionInput` class.
-* Deleted the deprecated `isOutParameterPointer`, `isOutQualifier`, `isOutReturnValue`, and `isOutReturnPointer` predicate from the `FunctionOutput` class.
-* Deleted the deprecated 3-argument `isGuardPhi` predicate from the `RangeSsaDefinition` class.

cpp/ql/lib/change-notes/released/0.6.1.md

@@ -1,3 +0,0 @@
-## 0.6.1
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.7.0.md

@@ -1,25 +0,0 @@
-## 0.7.0
-
-### Breaking Changes
-
-* The internal `SsaConsistency` module has been moved from `SSAConstruction` to `SSAConsitency`, and the deprecated `SSAConsistency` module has been removed.
-
-### Deprecated APIs
-
-* The single-parameter predicates `ArrayOrVectorAggregateLiteral.getElementExpr` and `ClassAggregateLiteral.getFieldExpr` have been deprecated in favor of `ArrayOrVectorAggregateLiteral.getAnElementExpr` and `ClassAggregateLiteral.getAFieldExpr`.
-* The recently introduced new data flow and taint tracking APIs have had a
-  number of module and predicate renamings. The old APIs remain in place for
-  now.
-* The `SslContextCallAbstractConfig`, `SslContextCallConfig`, `SslContextCallBannedProtocolConfig`, `SslContextCallTls12ProtocolConfig`, `SslContextCallTls13ProtocolConfig`, `SslContextCallTlsProtocolConfig`, `SslContextFlowsToSetOptionConfig`, `SslOptionConfig` dataflow configurations from `BoostorgAsio` have been deprecated. Please use `SslContextCallConfigSig`, `SslContextCallGlobal`, `SslContextCallFlow`, `SslContextCallBannedProtocolFlow`, `SslContextCallTls12ProtocolFlow`, `SslContextCallTls13ProtocolFlow`, `SslContextCallTlsProtocolFlow`, `SslContextFlowsToSetOptionFlow`.
-
-### New Features
-
-* Added overridable predicates `getSizeExpr` and `getSizeMult` to the `BufferAccess` class (`semmle.code.cpp.security.BufferAccess.qll`). This makes it possible to model a larger class of buffer reads and writes using the library.
-
-### Minor Analysis Improvements
-
-* The `BufferAccess` library (`semmle.code.cpp.security.BufferAccess`) no longer matches buffer accesses inside unevaluated contexts (such as inside `sizeof` or `decltype` expressions). As a result, queries using this library may see fewer false positives.
-
-### Bug Fixes
-
-* Fixed some accidental predicate visibility in the backwards-compatible wrapper for data flow configurations. In particular `DataFlow::hasFlowPath`, `DataFlow::hasFlow`, `DataFlow::hasFlowTo`, and `DataFlow::hasFlowToExpr` were accidentally exposed in a single version.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.7.0
+lastReleaseVersion: 0.4.2

cpp/ql/lib/definitions.qll

@@ -12,8 +12,8 @@ import IDEContextual
  *
  * In some cases it is preferable to modify locations (the
  * `hasLocationInfo()` predicate) so that they are short, and
- * non-overlapping with other locations that might be reported as
- * code scanning alerts on GitHub.
+ * non-overlapping with other locations that might be highlighted in
+ * the LGTM interface.
  *
  * We need to give locations that may not be in the database, so
  * we use `hasLocationInfo()` rather than `getLocation()`.
@@ -123,13 +123,6 @@ private predicate constructorCallTypeMention(ConstructorCall cc, TypeMention tm)
   )
 }
 
-/** Holds if `loc` has the container `container` and is on the line starting at `startLine`. */
-pragma[nomagic]
-private predicate hasContainerAndStartLine(Location loc, Container container, int startLine) {
-  loc.getStartLine() = startLine and
-  loc.getContainer() = container
-}
-
 /**
  * Gets an element, of kind `kind`, that element `e` uses, if any.
  * Attention: This predicate yields multiple definitions for a single location.
@@ -166,9 +159,9 @@ Top definitionOf(Top e, string kind) {
     // Multiple type mentions can be generated when a typedef is used, and
     // in such cases we want to exclude all but the originating typedef.
     not exists(Type secondary |
-      exists(File f, int startline, int startcol |
+      exists(TypeMention tm, File f, int startline, int startcol |
         typeMentionStartLoc(e, result, f, startline, startcol) and
-        typeMentionStartLoc(_, secondary, f, startline, startcol) and
+        typeMentionStartLoc(tm, secondary, f, startline, startcol) and
         (
           result = secondary.(TypedefType).getBaseType() or
           result = secondary.(TypedefType).getBaseType().(SpecifiedType).getBaseType()
@@ -191,9 +184,11 @@ Top definitionOf(Top e, string kind) {
     kind = "I" and
     result = e.(Include).getIncludedFile() and
     // exclude `#include` directives containing macros
-    not exists(MacroInvocation mi, Container container, int startLine |
-      hasContainerAndStartLine(e.(Include).getLocation(), container, startLine) and
-      hasContainerAndStartLine(mi.getLocation(), container, startLine)
+    not exists(MacroInvocation mi, Location l1, Location l2 |
+      l1 = e.(Include).getLocation() and
+      l2 = mi.getLocation() and
+      l1.getContainer() = l2.getContainer() and
+      l1.getStartLine() = l2.getStartLine()
       // (an #include directive must be always on it's own line)
     )
   ) and

cpp/ql/lib/experimental/semmle/code/cpp/dataflow/ProductFlow.qll

@@ -1,146 +1,17 @@
-import semmle.code.cpp.ir.dataflow.DataFlow
-private import codeql.util.Unit
+import experimental.semmle.code.cpp.ir.dataflow.DataFlow
+import experimental.semmle.code.cpp.ir.dataflow.DataFlow2
 
 module ProductFlow {
-  signature module ConfigSig {
+  abstract class Configuration extends string {
+    bindingset[this]
+    Configuration() { any() }
+
     /**
      * Holds if `(source1, source2)` is a relevant data flow source.
      *
      * `source1` and `source2` must belong to the same callable.
      */
-    predicate isSourcePair(DataFlow::Node source1, DataFlow::Node source2);
-
-    /**
-     * Holds if `(sink1, sink2)` is a relevant data flow sink.
-     *
-     * `sink1` and `sink2` must belong to the same callable.
-     */
-    predicate isSinkPair(DataFlow::Node sink1, DataFlow::Node sink2);
-
-    /**
-     * Holds if data flow through `node` is prohibited through the first projection of the product
-     * dataflow graph.
-     */
-    default predicate isBarrier1(DataFlow::Node node) { none() }
-
-    /**
-     * Holds if data flow through `node` is prohibited through the second projection of the product
-     * dataflow graph.
-     */
-    default predicate isBarrier2(DataFlow::Node node) { none() }
-
-    /**
-     * Holds if data flow out of `node` is prohibited in the first projection of the product
-     * dataflow graph.
-     */
-    default predicate isBarrierOut1(DataFlow::Node node) { none() }
-
-    /**
-     * Holds if data flow out of `node` is prohibited in the second projection of the product
-     * dataflow graph.
-     */
-    default predicate isBarrierOut2(DataFlow::Node node) { none() }
-
-    /*
-     * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps in
-     * the first projection of the product dataflow graph.
-     */
-
-    default predicate isAdditionalFlowStep1(DataFlow::Node node1, DataFlow::Node node2) { none() }
-
-    /**
-     * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps in
-     * the second projection of the product dataflow graph.
-     */
-    default predicate isAdditionalFlowStep2(DataFlow::Node node1, DataFlow::Node node2) { none() }
-
-    /**
-     * Holds if data flow into `node` is prohibited in the first projection of the product
-     * dataflow graph.
-     */
-    default predicate isBarrierIn1(DataFlow::Node node) { none() }
-
-    /**
-     * Holds if data flow into `node` is prohibited in the second projection of the product
-     * dataflow graph.
-     */
-    default predicate isBarrierIn2(DataFlow::Node node) { none() }
-  }
-
-  module Global<ConfigSig Config> {
-    private module StateConfig implements StateConfigSig {
-      class FlowState1 = Unit;
-
-      class FlowState2 = Unit;
-
-      predicate isSourcePair(
-        DataFlow::Node source1, FlowState1 state1, DataFlow::Node source2, FlowState2 state2
-      ) {
-        exists(state1) and
-        exists(state2) and
-        Config::isSourcePair(source1, source2)
-      }
-
-      predicate isSinkPair(
-        DataFlow::Node sink1, FlowState1 state1, DataFlow::Node sink2, FlowState2 state2
-      ) {
-        exists(state1) and
-        exists(state2) and
-        Config::isSinkPair(sink1, sink2)
-      }
-
-      predicate isBarrier1(DataFlow::Node node, FlowState1 state) {
-        exists(state) and
-        Config::isBarrier1(node)
-      }
-
-      predicate isBarrier2(DataFlow::Node node, FlowState2 state) {
-        exists(state) and
-        Config::isBarrier2(node)
-      }
-
-      predicate isBarrier1 = Config::isBarrier1/1;
-
-      predicate isBarrier2 = Config::isBarrier2/1;
-
-      predicate isBarrierOut1 = Config::isBarrierOut1/1;
-
-      predicate isBarrierOut2 = Config::isBarrierOut2/1;
-
-      predicate isAdditionalFlowStep1 = Config::isAdditionalFlowStep1/2;
-
-      predicate isAdditionalFlowStep1(
-        DataFlow::Node node1, FlowState1 state1, DataFlow::Node node2, FlowState1 state2
-      ) {
-        exists(state1) and
-        exists(state2) and
-        Config::isAdditionalFlowStep1(node1, node2)
-      }
-
-      predicate isAdditionalFlowStep2 = Config::isAdditionalFlowStep2/2;
-
-      predicate isAdditionalFlowStep2(
-        DataFlow::Node node1, FlowState2 state1, DataFlow::Node node2, FlowState2 state2
-      ) {
-        exists(state1) and
-        exists(state2) and
-        Config::isAdditionalFlowStep2(node1, node2)
-      }
-
-      predicate isBarrierIn1 = Config::isBarrierIn1/1;
-
-      predicate isBarrierIn2 = Config::isBarrierIn2/1;
-    }
-
-    import GlobalWithState<StateConfig>
-  }
-
-  signature module StateConfigSig {
-    bindingset[this]
-    class FlowState1;
-
-    bindingset[this]
-    class FlowState2;
+    predicate isSourcePair(DataFlow::Node source1, DataFlow::Node source2) { none() }
 
     /**
      * Holds if `(source1, source2)` is a relevant data flow source with initial states `state1`
@@ -149,8 +20,20 @@ module ProductFlow {
      * `source1` and `source2` must belong to the same callable.
      */
     predicate isSourcePair(
-      DataFlow::Node source1, FlowState1 state1, DataFlow::Node source2, FlowState2 state2
-    );
+      DataFlow::Node source1, DataFlow::FlowState state1, DataFlow::Node source2,
+      DataFlow::FlowState state2
+    ) {
+      state1 = "" and
+      state2 = "" and
+      this.isSourcePair(source1, source2)
+    }
+
+    /**
+     * Holds if `(sink1, sink2)` is a relevant data flow sink.
+     *
+     * `sink1` and `sink2` must belong to the same callable.
+     */
+    predicate isSinkPair(DataFlow::Node sink1, DataFlow::Node sink2) { none() }
 
     /**
      * Holds if `(sink1, sink2)` is a relevant data flow sink with final states `state1`
@@ -159,51 +42,60 @@ module ProductFlow {
      * `sink1` and `sink2` must belong to the same callable.
      */
     predicate isSinkPair(
-      DataFlow::Node sink1, FlowState1 state1, DataFlow::Node sink2, FlowState2 state2
-    );
+      DataFlow::Node sink1, DataFlow::FlowState state1, DataFlow::Node sink2,
+      DataFlow::FlowState state2
+    ) {
+      state1 = "" and
+      state2 = "" and
+      this.isSinkPair(sink1, sink2)
+    }
 
     /**
      * Holds if data flow through `node` is prohibited through the first projection of the product
      * dataflow graph when the flow state is `state`.
      */
-    predicate isBarrier1(DataFlow::Node node, FlowState1 state);
+    predicate isBarrier1(DataFlow::Node node, DataFlow::FlowState state) {
+      this.isBarrier1(node) and state = ""
+    }
 
     /**
      * Holds if data flow through `node` is prohibited through the second projection of the product
      * dataflow graph when the flow state is `state`.
      */
-    predicate isBarrier2(DataFlow::Node node, FlowState2 state);
+    predicate isBarrier2(DataFlow::Node node, DataFlow::FlowState state) {
+      this.isBarrier2(node) and state = ""
+    }
 
     /**
      * Holds if data flow through `node` is prohibited through the first projection of the product
      * dataflow graph.
      */
-    default predicate isBarrier1(DataFlow::Node node) { none() }
+    predicate isBarrier1(DataFlow::Node node) { none() }
 
     /**
      * Holds if data flow through `node` is prohibited through the second projection of the product
      * dataflow graph.
      */
-    default predicate isBarrier2(DataFlow::Node node) { none() }
+    predicate isBarrier2(DataFlow::Node node) { none() }
 
     /**
      * Holds if data flow out of `node` is prohibited in the first projection of the product
      * dataflow graph.
      */
-    default predicate isBarrierOut1(DataFlow::Node node) { none() }
+    predicate isBarrierOut1(DataFlow::Node node) { none() }
 
     /**
      * Holds if data flow out of `node` is prohibited in the second projection of the product
      * dataflow graph.
      */
-    default predicate isBarrierOut2(DataFlow::Node node) { none() }
+    predicate isBarrierOut2(DataFlow::Node node) { none() }
 
     /*
      * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps in
      * the first projection of the product dataflow graph.
      */
 
-    default predicate isAdditionalFlowStep1(DataFlow::Node node1, DataFlow::Node node2) { none() }
+    predicate isAdditionalFlowStep1(DataFlow::Node node1, DataFlow::Node node2) { none() }
 
     /**
      * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps in
@@ -212,14 +104,19 @@ module ProductFlow {
      * This step is only applicable in `state1` and updates the flow state to `state2`.
      */
     predicate isAdditionalFlowStep1(
-      DataFlow::Node node1, FlowState1 state1, DataFlow::Node node2, FlowState1 state2
-    );
+      DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,
+      DataFlow::FlowState state2
+    ) {
+      state1 instanceof DataFlow::FlowStateEmpty and
+      state2 instanceof DataFlow::FlowStateEmpty and
+      this.isAdditionalFlowStep1(node1, node2)
+    }
 
     /**
      * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps in
      * the second projection of the product dataflow graph.
      */
-    default predicate isAdditionalFlowStep2(DataFlow::Node node1, DataFlow::Node node2) { none() }
+    predicate isAdditionalFlowStep2(DataFlow::Node node1, DataFlow::Node node2) { none() }
 
     /**
      * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps in
@@ -228,168 +125,177 @@ module ProductFlow {
      * This step is only applicable in `state1` and updates the flow state to `state2`.
      */
     predicate isAdditionalFlowStep2(
-      DataFlow::Node node1, FlowState2 state1, DataFlow::Node node2, FlowState2 state2
-    );
+      DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,
+      DataFlow::FlowState state2
+    ) {
+      state1 instanceof DataFlow::FlowStateEmpty and
+      state2 instanceof DataFlow::FlowStateEmpty and
+      this.isAdditionalFlowStep2(node1, node2)
+    }
 
     /**
      * Holds if data flow into `node` is prohibited in the first projection of the product
      * dataflow graph.
      */
-    default predicate isBarrierIn1(DataFlow::Node node) { none() }
+    predicate isBarrierIn1(DataFlow::Node node) { none() }
 
     /**
      * Holds if data flow into `node` is prohibited in the second projection of the product
      * dataflow graph.
      */
-    default predicate isBarrierIn2(DataFlow::Node node) { none() }
+    predicate isBarrierIn2(DataFlow::Node node) { none() }
+
+    predicate hasFlowPath(
+      DataFlow::PathNode source1, DataFlow2::PathNode source2, DataFlow::PathNode sink1,
+      DataFlow2::PathNode sink2
+    ) {
+      reachable(this, source1, source2, sink1, sink2)
+    }
   }
 
-  module GlobalWithState<StateConfigSig Config> {
-    class PathNode1 = Flow1::PathNode;
+  private import Internal
 
-    class PathNode2 = Flow2::PathNode;
+  module Internal {
+    class Conf1 extends DataFlow::Configuration {
+      Conf1() { this = "Conf1" }
 
-    module PathGraph1 = Flow1::PathGraph;
-
-    module PathGraph2 = Flow2::PathGraph;
-
-    class FlowState1 = Config::FlowState1;
-
-    class FlowState2 = Config::FlowState2;
-
-    predicate flowPath(
-      Flow1::PathNode source1, Flow2::PathNode source2, Flow1::PathNode sink1, Flow2::PathNode sink2
-    ) {
-      reachable(source1, source2, sink1, sink2)
-    }
-
-    private module Config1 implements DataFlow::StateConfigSig {
-      class FlowState = FlowState1;
-
-      predicate isSource(DataFlow::Node source, FlowState state) {
-        Config::isSourcePair(source, state, _, _)
+      override predicate isSource(DataFlow::Node source, DataFlow::FlowState state) {
+        exists(Configuration conf | conf.isSourcePair(source, state, _, _))
       }
 
-      predicate isSink(DataFlow::Node sink, FlowState state) {
-        Config::isSinkPair(sink, state, _, _)
+      override predicate isSink(DataFlow::Node sink, DataFlow::FlowState state) {
+        exists(Configuration conf | conf.isSinkPair(sink, state, _, _))
       }
 
-      predicate isBarrier(DataFlow::Node node, FlowState state) { Config::isBarrier1(node, state) }
+      override predicate isBarrier(DataFlow::Node node, DataFlow::FlowState state) {
+        exists(Configuration conf | conf.isBarrier1(node, state))
+      }
 
-      predicate isBarrierOut(DataFlow::Node node) { Config::isBarrierOut1(node) }
+      override predicate isBarrierOut(DataFlow::Node node) {
+        exists(Configuration conf | conf.isBarrierOut1(node))
+      }
 
-      predicate isAdditionalFlowStep(
-        DataFlow::Node node1, FlowState1 state1, DataFlow::Node node2, FlowState state2
+      override predicate isAdditionalFlowStep(
+        DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,
+        DataFlow::FlowState state2
       ) {
-        Config::isAdditionalFlowStep1(node1, state1, node2, state2)
+        exists(Configuration conf | conf.isAdditionalFlowStep1(node1, state1, node2, state2))
       }
 
-      predicate isBarrierIn(DataFlow::Node node) { Config::isBarrierIn1(node) }
+      override predicate isBarrierIn(DataFlow::Node node) {
+        exists(Configuration conf | conf.isBarrierIn1(node))
+      }
     }
 
-    module Flow1 = DataFlow::GlobalWithState<Config1>;
+    class Conf2 extends DataFlow2::Configuration {
+      Conf2() { this = "Conf2" }
 
-    module Config2 implements DataFlow::StateConfigSig {
-      class FlowState = FlowState2;
-
-      predicate isSource(DataFlow::Node source, FlowState state) {
-        exists(Flow1::PathNode source1 |
-          Config::isSourcePair(source1.getNode(), source1.getState(), source, state) and
-          Flow1::flowPath(source1, _)
+      override predicate isSource(DataFlow::Node source, DataFlow::FlowState state) {
+        exists(Configuration conf, DataFlow::PathNode source1 |
+          conf.isSourcePair(source1.getNode(), source1.getState(), source, state) and
+          any(Conf1 c).hasFlowPath(source1, _)
         )
       }
 
-      predicate isSink(DataFlow::Node sink, FlowState state) {
-        exists(Flow1::PathNode sink1 |
-          Config::isSinkPair(sink1.getNode(), sink1.getState(), sink, state) and
-          Flow1::flowPath(_, sink1)
+      override predicate isSink(DataFlow::Node sink, DataFlow::FlowState state) {
+        exists(Configuration conf, DataFlow::PathNode sink1 |
+          conf.isSinkPair(sink1.getNode(), sink1.getState(), sink, state) and
+          any(Conf1 c).hasFlowPath(_, sink1)
         )
       }
 
-      predicate isBarrier(DataFlow::Node node, FlowState state) { Config::isBarrier2(node, state) }
-
-      predicate isBarrierOut(DataFlow::Node node) { Config::isBarrierOut2(node) }
-
-      predicate isAdditionalFlowStep(
-        DataFlow::Node node1, FlowState state1, DataFlow::Node node2, FlowState state2
-      ) {
-        Config::isAdditionalFlowStep2(node1, state1, node2, state2)
+      override predicate isBarrier(DataFlow::Node node, DataFlow::FlowState state) {
+        exists(Configuration conf | conf.isBarrier2(node, state))
       }
 
-      predicate isBarrierIn(DataFlow::Node node) { Config::isBarrierIn2(node) }
-    }
+      override predicate isBarrierOut(DataFlow::Node node) {
+        exists(Configuration conf | conf.isBarrierOut2(node))
+      }
 
-    module Flow2 = DataFlow::GlobalWithState<Config2>;
+      override predicate isAdditionalFlowStep(
+        DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,
+        DataFlow::FlowState state2
+      ) {
+        exists(Configuration conf | conf.isAdditionalFlowStep2(node1, state1, node2, state2))
+      }
 
-    pragma[nomagic]
-    private predicate reachableInterprocEntry(
-      Flow1::PathNode source1, Flow2::PathNode source2, Flow1::PathNode node1, Flow2::PathNode node2
-    ) {
-      Config::isSourcePair(node1.getNode(), node1.getState(), node2.getNode(), node2.getState()) and
-      node1 = source1 and
-      node2 = source2
-      or
-      exists(
-        Flow1::PathNode midEntry1, Flow2::PathNode midEntry2, Flow1::PathNode midExit1,
-        Flow2::PathNode midExit2
-      |
-        reachableInterprocEntry(source1, source2, midEntry1, midEntry2) and
-        interprocEdgePair(midExit1, midExit2, node1, node2) and
-        localPathStep1*(midEntry1, midExit1) and
-        localPathStep2*(midEntry2, midExit2)
-      )
+      override predicate isBarrierIn(DataFlow::Node node) {
+        exists(Configuration conf | conf.isBarrierIn2(node))
+      }
     }
+  }
 
-    private predicate localPathStep1(Flow1::PathNode pred, Flow1::PathNode succ) {
-      Flow1::PathGraph::edges(pred, succ) and
-      pragma[only_bind_out](pred.getNode().getEnclosingCallable()) =
-        pragma[only_bind_out](succ.getNode().getEnclosingCallable())
-    }
+  pragma[nomagic]
+  private predicate reachableInterprocEntry(
+    Configuration conf, DataFlow::PathNode source1, DataFlow2::PathNode source2,
+    DataFlow::PathNode node1, DataFlow2::PathNode node2
+  ) {
+    conf.isSourcePair(node1.getNode(), node1.getState(), node2.getNode(), node2.getState()) and
+    node1 = source1 and
+    node2 = source2
+    or
+    exists(
+      DataFlow::PathNode midEntry1, DataFlow2::PathNode midEntry2, DataFlow::PathNode midExit1,
+      DataFlow2::PathNode midExit2
+    |
+      reachableInterprocEntry(conf, source1, source2, midEntry1, midEntry2) and
+      interprocEdgePair(midExit1, midExit2, node1, node2) and
+      localPathStep1*(midEntry1, midExit1) and
+      localPathStep2*(midEntry2, midExit2)
+    )
+  }
 
-    private predicate localPathStep2(Flow2::PathNode pred, Flow2::PathNode succ) {
-      Flow2::PathGraph::edges(pred, succ) and
-      pragma[only_bind_out](pred.getNode().getEnclosingCallable()) =
-        pragma[only_bind_out](succ.getNode().getEnclosingCallable())
-    }
+  private predicate localPathStep1(DataFlow::PathNode pred, DataFlow::PathNode succ) {
+    DataFlow::PathGraph::edges(pred, succ) and
+    pragma[only_bind_out](pred.getNode().getEnclosingCallable()) =
+      pragma[only_bind_out](succ.getNode().getEnclosingCallable())
+  }
 
-    pragma[nomagic]
-    private predicate interprocEdge1(
-      Declaration predDecl, Declaration succDecl, Flow1::PathNode pred1, Flow1::PathNode succ1
-    ) {
-      Flow1::PathGraph::edges(pred1, succ1) and
-      predDecl != succDecl and
-      pred1.getNode().getEnclosingCallable() = predDecl and
-      succ1.getNode().getEnclosingCallable() = succDecl
-    }
+  private predicate localPathStep2(DataFlow2::PathNode pred, DataFlow2::PathNode succ) {
+    DataFlow2::PathGraph::edges(pred, succ) and
+    pragma[only_bind_out](pred.getNode().getEnclosingCallable()) =
+      pragma[only_bind_out](succ.getNode().getEnclosingCallable())
+  }
 
-    pragma[nomagic]
-    private predicate interprocEdge2(
-      Declaration predDecl, Declaration succDecl, Flow2::PathNode pred2, Flow2::PathNode succ2
-    ) {
-      Flow2::PathGraph::edges(pred2, succ2) and
-      predDecl != succDecl and
-      pred2.getNode().getEnclosingCallable() = predDecl and
-      succ2.getNode().getEnclosingCallable() = succDecl
-    }
+  pragma[nomagic]
+  private predicate interprocEdge1(
+    Declaration predDecl, Declaration succDecl, DataFlow::PathNode pred1, DataFlow::PathNode succ1
+  ) {
+    DataFlow::PathGraph::edges(pred1, succ1) and
+    predDecl != succDecl and
+    pred1.getNode().getEnclosingCallable() = predDecl and
+    succ1.getNode().getEnclosingCallable() = succDecl
+  }
 
-    private predicate interprocEdgePair(
-      Flow1::PathNode pred1, Flow2::PathNode pred2, Flow1::PathNode succ1, Flow2::PathNode succ2
-    ) {
-      exists(Declaration predDecl, Declaration succDecl |
-        interprocEdge1(predDecl, succDecl, pred1, succ1) and
-        interprocEdge2(predDecl, succDecl, pred2, succ2)
-      )
-    }
+  pragma[nomagic]
+  private predicate interprocEdge2(
+    Declaration predDecl, Declaration succDecl, DataFlow2::PathNode pred2, DataFlow2::PathNode succ2
+  ) {
+    DataFlow2::PathGraph::edges(pred2, succ2) and
+    predDecl != succDecl and
+    pred2.getNode().getEnclosingCallable() = predDecl and
+    succ2.getNode().getEnclosingCallable() = succDecl
+  }
 
-    private predicate reachable(
-      Flow1::PathNode source1, Flow2::PathNode source2, Flow1::PathNode sink1, Flow2::PathNode sink2
-    ) {
-      exists(Flow1::PathNode mid1, Flow2::PathNode mid2 |
-        reachableInterprocEntry(source1, source2, mid1, mid2) and
-        Config::isSinkPair(sink1.getNode(), sink1.getState(), sink2.getNode(), sink2.getState()) and
-        localPathStep1*(mid1, sink1) and
-        localPathStep2*(mid2, sink2)
-      )
-    }
+  private predicate interprocEdgePair(
+    DataFlow::PathNode pred1, DataFlow2::PathNode pred2, DataFlow::PathNode succ1,
+    DataFlow2::PathNode succ2
+  ) {
+    exists(Declaration predDecl, Declaration succDecl |
+      interprocEdge1(predDecl, succDecl, pred1, succ1) and
+      interprocEdge2(predDecl, succDecl, pred2, succ2)
+    )
+  }
+
+  private predicate reachable(
+    Configuration conf, DataFlow::PathNode source1, DataFlow2::PathNode source2,
+    DataFlow::PathNode sink1, DataFlow2::PathNode sink2
+  ) {
+    exists(DataFlow::PathNode mid1, DataFlow2::PathNode mid2 |
+      reachableInterprocEntry(conf, source1, source2, mid1, mid2) and
+      conf.isSinkPair(sink1.getNode(), sink1.getState(), sink2.getNode(), sink2.getState()) and
+      localPathStep1*(mid1, sink1) and
+      localPathStep2*(mid2, sink2)
+    )
   }
 }

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/DataFlow.qll

@@ -0,0 +1,26 @@
+/**
+ * Provides a library for local (intra-procedural) and global (inter-procedural)
+ * data flow analysis: deciding whether data can flow from a _source_ to a
+ * _sink_. This library differs from the one in `semmle.code.cpp.dataflow` in that
+ * this library uses the IR (Intermediate Representation) library, which provides
+ * a more precise semantic representation of the program, whereas the other dataflow
+ * library uses the more syntax-oriented ASTs. This library should provide more accurate
+ * results than the AST-based library in most scenarios.
+ *
+ * Unless configured otherwise, _flow_ means that the exact value of
+ * the source may reach the sink. We do not track flow across pointer
+ * dereferences or array indexing.
+ *
+ * To use global (interprocedural) data flow, extend the class
+ * `DataFlow::Configuration` as documented on that class. To use local
+ * (intraprocedural) data flow between expressions, call
+ * `DataFlow::localExprFlow`. For more general cases of local data flow, call
+ * `DataFlow::localFlow` or `DataFlow::localFlowStep` with arguments of type
+ * `DataFlow::Node`.
+ */
+
+import cpp
+
+module DataFlow {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/DataFlow2.qll

@@ -0,0 +1,16 @@
+/**
+ * Provides a `DataFlow2` module, which is a copy of the `DataFlow` module. Use
+ * this class when data-flow configurations must depend on each other. Two
+ * classes extending `DataFlow::Configuration` should never depend on each
+ * other, but one of them should instead depend on a
+ * `DataFlow2::Configuration`, a `DataFlow3::Configuration`, or a
+ * `DataFlow4::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.DataFlow` for the full documentation.
+ */
+
+import cpp
+
+module DataFlow2 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl2
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/DataFlow3.qll

@@ -0,0 +1,16 @@
+/**
+ * Provides a `DataFlow3` module, which is a copy of the `DataFlow` module. Use
+ * this class when data-flow configurations must depend on each other. Two
+ * classes extending `DataFlow::Configuration` should never depend on each
+ * other, but one of them should instead depend on a
+ * `DataFlow2::Configuration`, a `DataFlow3::Configuration`, or a
+ * `DataFlow4::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.DataFlow` for the full documentation.
+ */
+
+import cpp
+
+module DataFlow3 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl3
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/DataFlow4.qll

@@ -0,0 +1,16 @@
+/**
+ * Provides a `DataFlow4` module, which is a copy of the `DataFlow` module. Use
+ * this class when data-flow configurations must depend on each other. Two
+ * classes extending `DataFlow::Configuration` should never depend on each
+ * other, but one of them should instead depend on a
+ * `DataFlow2::Configuration`, a `DataFlow3::Configuration`, or a
+ * `DataFlow4::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.DataFlow` for the full documentation.
+ */
+
+import cpp
+
+module DataFlow4 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl4
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/ResolveCall.qll

@@ -0,0 +1,23 @@
+/**
+ * Provides a predicate for non-contextual virtual dispatch and function
+ * pointer resolution.
+ */
+
+import cpp
+private import semmle.code.cpp.ir.ValueNumbering
+private import internal.DataFlowDispatch
+private import semmle.code.cpp.ir.IR
+
+/**
+ * Resolve potential target function(s) for `call`.
+ *
+ * If `call` is a call through a function pointer (`ExprCall`) or its target is
+ * a virtual member function, simple data flow analysis is performed in order
+ * to identify the possible target(s).
+ */
+Function resolveCall(Call call) {
+  exists(CallInstruction callInstruction |
+    callInstruction.getAst() = call and
+    result = viableCallable(callInstruction)
+  )
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/TaintTracking.qll

@@ -0,0 +1,23 @@
+/**
+ * Provides classes for performing local (intra-procedural) and
+ * global (inter-procedural) taint-tracking analyses.
+ *
+ * We define _taint propagation_ informally to mean that a substantial part of
+ * the information from the source is preserved at the sink. For example, taint
+ * propagates from `x` to `x + 100`, but it does not propagate from `x` to `x >
+ * 100` since we consider a single bit of information to be too little.
+ *
+ * To use global (interprocedural) taint tracking, extend the class
+ * `TaintTracking::Configuration` as documented on that class. To use local
+ * (intraprocedural) taint tracking between expressions, call
+ * `TaintTracking::localExprTaint`. For more general cases of local taint
+ * tracking, call `TaintTracking::localTaint` or
+ * `TaintTracking::localTaintStep` with arguments of type `DataFlow::Node`.
+ */
+
+import semmle.code.cpp.ir.dataflow.DataFlow
+import semmle.code.cpp.ir.dataflow.DataFlow2
+
+module TaintTracking {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.tainttracking1.TaintTrackingImpl
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/TaintTracking2.qll

@@ -0,0 +1,15 @@
+/**
+ * Provides a `TaintTracking2` module, which is a copy of the `TaintTracking`
+ * module. Use this class when data-flow configurations or taint-tracking
+ * configurations must depend on each other. Two classes extending
+ * `DataFlow::Configuration` should never depend on each other, but one of them
+ * should instead depend on a `DataFlow2::Configuration`, a
+ * `DataFlow3::Configuration`, or a `DataFlow4::Configuration`. The
+ * `TaintTracking::Configuration` class extends `DataFlow::Configuration`, and
+ * `TaintTracking2::Configuration` extends `DataFlow2::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.TaintTracking` for the full documentation.
+ */
+module TaintTracking2 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.tainttracking2.TaintTrackingImpl
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/TaintTracking3.qll

@@ -0,0 +1,15 @@
+/**
+ * Provides a `TaintTracking3` module, which is a copy of the `TaintTracking`
+ * module. Use this class when data-flow configurations or taint-tracking
+ * configurations must depend on each other. Two classes extending
+ * `DataFlow::Configuration` should never depend on each other, but one of them
+ * should instead depend on a `DataFlow2::Configuration`, a
+ * `DataFlow3::Configuration`, or a `DataFlow4::Configuration`. The
+ * `TaintTracking::Configuration` class extends `DataFlow::Configuration`, and
+ * `TaintTracking2::Configuration` extends `DataFlow2::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.TaintTracking` for the full documentation.
+ */
+module TaintTracking3 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.tainttracking3.TaintTrackingImpl
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll

@@ -0,0 +1,273 @@
+private import cpp
+private import semmle.code.cpp.ir.IR
+private import experimental.semmle.code.cpp.ir.dataflow.DataFlow
+private import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowPrivate
+private import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowUtil
+private import DataFlowImplCommon as DataFlowImplCommon
+
+/**
+ * Gets a function that might be called by `call`.
+ */
+cached
+Function viableCallable(CallInstruction call) {
+  DataFlowImplCommon::forceCachingInSameStage() and
+  result = call.getStaticCallTarget()
+  or
+  // If the target of the call does not have a body in the snapshot, it might
+  // be because the target is just a header declaration, and the real target
+  // will be determined at run time when the caller and callee are linked
+  // together by the operating system's dynamic linker. In case a _unique_
+  // function with the right signature is present in the database, we return
+  // that as a potential callee.
+  exists(string qualifiedName, int nparams |
+    callSignatureWithoutBody(qualifiedName, nparams, call) and
+    functionSignatureWithBody(qualifiedName, nparams, result) and
+    strictcount(Function other | functionSignatureWithBody(qualifiedName, nparams, other)) = 1
+  )
+  or
+  // Virtual dispatch
+  result = call.(VirtualDispatch::DataSensitiveCall).resolve()
+}
+
+/**
+ * Provides virtual dispatch support compatible with the original
+ * implementation of `semmle.code.cpp.security.TaintTracking`.
+ */
+private module VirtualDispatch {
+  /** A call that may dispatch differently depending on the qualifier value. */
+  abstract class DataSensitiveCall extends DataFlowCall {
+    /**
+     * Gets the node whose value determines the target of this call. This node
+     * could be the qualifier of a virtual dispatch or the function-pointer
+     * expression in a call to a function pointer. What they have in common is
+     * that we need to find out which data flows there, and then it's up to the
+     * `resolve` predicate to stitch that information together and resolve the
+     * call.
+     */
+    abstract DataFlow::Node getDispatchValue();
+
+    /** Gets a candidate target for this call. */
+    abstract Function resolve();
+
+    /**
+     * Whether `src` can flow to this call.
+     *
+     * Searches backwards from `getDispatchValue()` to `src`. The `allowFromArg`
+     * parameter is true when the search is allowed to continue backwards into
+     * a parameter; non-recursive callers should pass `_` for `allowFromArg`.
+     */
+    predicate flowsFrom(DataFlow::Node src, boolean allowFromArg) {
+      src = this.getDispatchValue() and allowFromArg = true
+      or
+      exists(DataFlow::Node other, boolean allowOtherFromArg |
+        this.flowsFrom(other, allowOtherFromArg)
+      |
+        // Call argument
+        exists(DataFlowCall call, Position i |
+          other
+              .(DataFlow::ParameterNode)
+              .isParameterOf(pragma[only_bind_into](call).getStaticCallTarget(), i) and
+          src.(ArgumentNode).argumentOf(call, pragma[only_bind_into](pragma[only_bind_out](i)))
+        ) and
+        allowOtherFromArg = true and
+        allowFromArg = true
+        or
+        // Call return
+        exists(DataFlowCall call, ReturnKind returnKind |
+          other = getAnOutNode(call, returnKind) and
+          returnNodeWithKindAndEnclosingCallable(src, returnKind, call.getStaticCallTarget())
+        ) and
+        allowFromArg = false
+        or
+        // Local flow
+        DataFlow::localFlowStep(src, other) and
+        allowFromArg = allowOtherFromArg
+        or
+        // Flow from global variable to load.
+        exists(LoadInstruction load, GlobalOrNamespaceVariable var |
+          var = src.asVariable() and
+          other.asInstruction() = load and
+          addressOfGlobal(load.getSourceAddress(), var) and
+          // The `allowFromArg` concept doesn't play a role when `src` is a
+          // global variable, so we just set it to a single arbitrary value for
+          // performance.
+          allowFromArg = true
+        )
+        or
+        // Flow from store to global variable.
+        exists(StoreInstruction store, GlobalOrNamespaceVariable var |
+          var = other.asVariable() and
+          store = src.asInstruction() and
+          storeIntoGlobal(store, var) and
+          // Setting `allowFromArg` to `true` like in the base case means we
+          // treat a store to a global variable like the dispatch itself: flow
+          // may come from anywhere.
+          allowFromArg = true
+        )
+      )
+    }
+  }
+
+  pragma[noinline]
+  private predicate storeIntoGlobal(StoreInstruction store, GlobalOrNamespaceVariable var) {
+    addressOfGlobal(store.getDestinationAddress(), var)
+  }
+
+  /** Holds if `addressInstr` is an instruction that produces the address of `var`. */
+  private predicate addressOfGlobal(Instruction addressInstr, GlobalOrNamespaceVariable var) {
+    // Access directly to the global variable
+    addressInstr.(VariableAddressInstruction).getAstVariable() = var
+    or
+    // Access to a field on a global union
+    exists(FieldAddressInstruction fa |
+      fa = addressInstr and
+      fa.getObjectAddress().(VariableAddressInstruction).getAstVariable() = var and
+      fa.getField().getDeclaringType() instanceof Union
+    )
+  }
+
+  /**
+   * A ReturnNode with its ReturnKind and its enclosing callable.
+   *
+   * Used to fix a join ordering issue in flowsFrom.
+   */
+  pragma[noinline]
+  private predicate returnNodeWithKindAndEnclosingCallable(
+    ReturnNode node, ReturnKind kind, DataFlowCallable callable
+  ) {
+    node.getKind() = kind and
+    node.getEnclosingCallable() = callable
+  }
+
+  /** Call through a function pointer. */
+  private class DataSensitiveExprCall extends DataSensitiveCall {
+    DataSensitiveExprCall() { not exists(this.getStaticCallTarget()) }
+
+    override DataFlow::Node getDispatchValue() { result.asInstruction() = this.getCallTarget() }
+
+    override Function resolve() {
+      exists(FunctionInstruction fi |
+        this.flowsFrom(DataFlow::instructionNode(fi), _) and
+        result = fi.getFunctionSymbol()
+      ) and
+      (
+        this.getNumberOfArguments() <= result.getEffectiveNumberOfParameters() and
+        this.getNumberOfArguments() >= result.getEffectiveNumberOfParameters()
+        or
+        result.isVarargs()
+      )
+    }
+  }
+
+  /** Call to a virtual function. */
+  private class DataSensitiveOverriddenFunctionCall extends DataSensitiveCall {
+    DataSensitiveOverriddenFunctionCall() {
+      exists(this.getStaticCallTarget().(VirtualFunction).getAnOverridingFunction())
+    }
+
+    override DataFlow::Node getDispatchValue() { result.asInstruction() = this.getThisArgument() }
+
+    override MemberFunction resolve() {
+      exists(Class overridingClass |
+        this.overrideMayAffectCall(overridingClass, result) and
+        this.hasFlowFromCastFrom(overridingClass)
+      )
+    }
+
+    /**
+     * Holds if `this` is a virtual function call whose static target is
+     * overridden by `overridingFunction` in `overridingClass`.
+     */
+    pragma[noinline]
+    private predicate overrideMayAffectCall(Class overridingClass, MemberFunction overridingFunction) {
+      overridingFunction.getAnOverriddenFunction+() = this.getStaticCallTarget().(VirtualFunction) and
+      overridingFunction.getDeclaringType() = overridingClass
+    }
+
+    /**
+     * Holds if the qualifier of `this` has flow from an upcast from
+     * `derivedClass`.
+     */
+    pragma[noinline]
+    private predicate hasFlowFromCastFrom(Class derivedClass) {
+      exists(ConvertToBaseInstruction toBase |
+        this.flowsFrom(DataFlow::instructionNode(toBase), _) and
+        derivedClass = toBase.getDerivedClass()
+      )
+    }
+  }
+}
+
+/**
+ * Holds if `f` is a function with a body that has name `qualifiedName` and
+ * `nparams` parameter count. See `functionSignature`.
+ */
+private predicate functionSignatureWithBody(string qualifiedName, int nparams, Function f) {
+  functionSignature(f, qualifiedName, nparams) and
+  exists(f.getBlock())
+}
+
+/**
+ * Holds if the target of `call` is a function _with no definition_ that has
+ * name `qualifiedName` and `nparams` parameter count. See `functionSignature`.
+ */
+pragma[noinline]
+private predicate callSignatureWithoutBody(string qualifiedName, int nparams, CallInstruction call) {
+  exists(Function target |
+    target = call.getStaticCallTarget() and
+    not exists(target.getBlock()) and
+    functionSignature(target, qualifiedName, nparams)
+  )
+}
+
+/**
+ * Holds if `f` has name `qualifiedName` and `nparams` parameter count. This is
+ * an approximation of its signature for the purpose of matching functions that
+ * might be the same across link targets.
+ */
+private predicate functionSignature(Function f, string qualifiedName, int nparams) {
+  qualifiedName = f.getQualifiedName() and
+  nparams = f.getNumberOfParameters() and
+  not f.isStatic()
+}
+
+/**
+ * Holds if the set of viable implementations that can be called by `call`
+ * might be improved by knowing the call context.
+ */
+predicate mayBenefitFromCallContext(CallInstruction call, Function f) {
+  mayBenefitFromCallContext(call, f, _)
+}
+
+/**
+ * Holds if `call` is a call through a function pointer, and the pointer
+ * value is given as the `arg`'th argument to `f`.
+ */
+private predicate mayBenefitFromCallContext(
+  VirtualDispatch::DataSensitiveCall call, Function f, int arg
+) {
+  f = pragma[only_bind_out](call).getEnclosingCallable() and
+  exists(InitializeParameterInstruction init |
+    not exists(call.getStaticCallTarget()) and
+    init.getEnclosingFunction() = f and
+    call.flowsFrom(DataFlow::instructionNode(init), _) and
+    init.getParameter().getIndex() = arg
+  )
+}
+
+/**
+ * Gets a viable dispatch target of `call` in the context `ctx`. This is
+ * restricted to those `call`s for which a context might make a difference.
+ */
+Function viableImplInCallContext(CallInstruction call, CallInstruction ctx) {
+  result = viableCallable(call) and
+  exists(int i, Function f |
+    mayBenefitFromCallContext(pragma[only_bind_into](call), f, i) and
+    f = ctx.getStaticCallTarget() and
+    result = ctx.getArgument(i).getUnconvertedResultExpression().(FunctionAccess).getTarget()
+  )
+}
+
+/** Holds if arguments at position `apos` match parameters at position `ppos`. */
+pragma[inline]
+predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) { ppos = apos }

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll

@@ -0,0 +1,235 @@
+/**
+ * Provides consistency queries for checking invariants in the language-specific
+ * data-flow classes and predicates.
+ */
+
+private import DataFlowImplSpecific::Private
+private import DataFlowImplSpecific::Public
+private import tainttracking1.TaintTrackingParameter::Private
+private import tainttracking1.TaintTrackingParameter::Public
+
+module Consistency {
+  private newtype TConsistencyConfiguration = MkConsistencyConfiguration()
+
+  /** A class for configuring the consistency queries. */
+  class ConsistencyConfiguration extends TConsistencyConfiguration {
+    string toString() { none() }
+
+    /** Holds if `n` should be excluded from the consistency test `uniqueEnclosingCallable`. */
+    predicate uniqueEnclosingCallableExclude(Node n) { none() }
+
+    /** Holds if `n` should be excluded from the consistency test `uniqueNodeLocation`. */
+    predicate uniqueNodeLocationExclude(Node n) { none() }
+
+    /** Holds if `n` should be excluded from the consistency test `missingLocation`. */
+    predicate missingLocationExclude(Node n) { none() }
+
+    /** Holds if `n` should be excluded from the consistency test `postWithInFlow`. */
+    predicate postWithInFlowExclude(Node n) { none() }
+
+    /** Holds if `n` should be excluded from the consistency test `argHasPostUpdate`. */
+    predicate argHasPostUpdateExclude(ArgumentNode n) { none() }
+
+    /** Holds if `n` should be excluded from the consistency test `reverseRead`. */
+    predicate reverseReadExclude(Node n) { none() }
+
+    /** Holds if `n` should be excluded from the consistency test `postHasUniquePre`. */
+    predicate postHasUniquePreExclude(PostUpdateNode n) { none() }
+
+    /** Holds if `n` should be excluded from the consistency test `uniquePostUpdate`. */
+    predicate uniquePostUpdateExclude(Node n) { none() }
+
+    /** Holds if `(call, ctx)` should be excluded from the consistency test `viableImplInCallContextTooLargeExclude`. */
+    predicate viableImplInCallContextTooLargeExclude(
+      DataFlowCall call, DataFlowCall ctx, DataFlowCallable callable
+    ) {
+      none()
+    }
+  }
+
+  private class RelevantNode extends Node {
+    RelevantNode() {
+      this instanceof ArgumentNode or
+      this instanceof ParameterNode or
+      this instanceof ReturnNode or
+      this = getAnOutNode(_, _) or
+      simpleLocalFlowStep(this, _) or
+      simpleLocalFlowStep(_, this) or
+      jumpStep(this, _) or
+      jumpStep(_, this) or
+      storeStep(this, _, _) or
+      storeStep(_, _, this) or
+      readStep(this, _, _) or
+      readStep(_, _, this) or
+      defaultAdditionalTaintStep(this, _) or
+      defaultAdditionalTaintStep(_, this)
+    }
+  }
+
+  query predicate uniqueEnclosingCallable(Node n, string msg) {
+    exists(int c |
+      n instanceof RelevantNode and
+      c = count(nodeGetEnclosingCallable(n)) and
+      c != 1 and
+      not any(ConsistencyConfiguration conf).uniqueEnclosingCallableExclude(n) and
+      msg = "Node should have one enclosing callable but has " + c + "."
+    )
+  }
+
+  query predicate uniqueType(Node n, string msg) {
+    exists(int c |
+      n instanceof RelevantNode and
+      c = count(getNodeType(n)) and
+      c != 1 and
+      msg = "Node should have one type but has " + c + "."
+    )
+  }
+
+  query predicate uniqueNodeLocation(Node n, string msg) {
+    exists(int c |
+      c =
+        count(string filepath, int startline, int startcolumn, int endline, int endcolumn |
+          n.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+        ) and
+      c != 1 and
+      not any(ConsistencyConfiguration conf).uniqueNodeLocationExclude(n) and
+      msg = "Node should have one location but has " + c + "."
+    )
+  }
+
+  query predicate missingLocation(string msg) {
+    exists(int c |
+      c =
+        strictcount(Node n |
+          not exists(string filepath, int startline, int startcolumn, int endline, int endcolumn |
+            n.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+          ) and
+          not any(ConsistencyConfiguration conf).missingLocationExclude(n)
+        ) and
+      msg = "Nodes without location: " + c
+    )
+  }
+
+  query predicate uniqueNodeToString(Node n, string msg) {
+    exists(int c |
+      c = count(n.toString()) and
+      c != 1 and
+      msg = "Node should have one toString but has " + c + "."
+    )
+  }
+
+  query predicate missingToString(string msg) {
+    exists(int c |
+      c = strictcount(Node n | not exists(n.toString())) and
+      msg = "Nodes without toString: " + c
+    )
+  }
+
+  query predicate parameterCallable(ParameterNode p, string msg) {
+    exists(DataFlowCallable c | isParameterNode(p, c, _) and c != nodeGetEnclosingCallable(p)) and
+    msg = "Callable mismatch for parameter."
+  }
+
+  query predicate localFlowIsLocal(Node n1, Node n2, string msg) {
+    simpleLocalFlowStep(n1, n2) and
+    nodeGetEnclosingCallable(n1) != nodeGetEnclosingCallable(n2) and
+    msg = "Local flow step does not preserve enclosing callable."
+  }
+
+  private DataFlowType typeRepr() { result = getNodeType(_) }
+
+  query predicate compatibleTypesReflexive(DataFlowType t, string msg) {
+    t = typeRepr() and
+    not compatibleTypes(t, t) and
+    msg = "Type compatibility predicate is not reflexive."
+  }
+
+  query predicate unreachableNodeCCtx(Node n, DataFlowCall call, string msg) {
+    isUnreachableInCall(n, call) and
+    exists(DataFlowCallable c |
+      c = nodeGetEnclosingCallable(n) and
+      not viableCallable(call) = c
+    ) and
+    msg = "Call context for isUnreachableInCall is inconsistent with call graph."
+  }
+
+  query predicate localCallNodes(DataFlowCall call, Node n, string msg) {
+    (
+      n = getAnOutNode(call, _) and
+      msg = "OutNode and call does not share enclosing callable."
+      or
+      n.(ArgumentNode).argumentOf(call, _) and
+      msg = "ArgumentNode and call does not share enclosing callable."
+    ) and
+    nodeGetEnclosingCallable(n) != call.getEnclosingCallable()
+  }
+
+  // This predicate helps the compiler forget that in some languages
+  // it is impossible for a result of `getPreUpdateNode` to be an
+  // instance of `PostUpdateNode`.
+  private Node getPre(PostUpdateNode n) {
+    result = n.getPreUpdateNode()
+    or
+    none()
+  }
+
+  query predicate postIsNotPre(PostUpdateNode n, string msg) {
+    getPre(n) = n and
+    msg = "PostUpdateNode should not equal its pre-update node."
+  }
+
+  query predicate postHasUniquePre(PostUpdateNode n, string msg) {
+    not any(ConsistencyConfiguration conf).postHasUniquePreExclude(n) and
+    exists(int c |
+      c = count(n.getPreUpdateNode()) and
+      c != 1 and
+      msg = "PostUpdateNode should have one pre-update node but has " + c + "."
+    )
+  }
+
+  query predicate uniquePostUpdate(Node n, string msg) {
+    not any(ConsistencyConfiguration conf).uniquePostUpdateExclude(n) and
+    1 < strictcount(PostUpdateNode post | post.getPreUpdateNode() = n) and
+    msg = "Node has multiple PostUpdateNodes."
+  }
+
+  query predicate postIsInSameCallable(PostUpdateNode n, string msg) {
+    nodeGetEnclosingCallable(n) != nodeGetEnclosingCallable(n.getPreUpdateNode()) and
+    msg = "PostUpdateNode does not share callable with its pre-update node."
+  }
+
+  private predicate hasPost(Node n) { exists(PostUpdateNode post | post.getPreUpdateNode() = n) }
+
+  query predicate reverseRead(Node n, string msg) {
+    exists(Node n2 | readStep(n, _, n2) and hasPost(n2) and not hasPost(n)) and
+    not any(ConsistencyConfiguration conf).reverseReadExclude(n) and
+    msg = "Origin of readStep is missing a PostUpdateNode."
+  }
+
+  query predicate argHasPostUpdate(ArgumentNode n, string msg) {
+    not hasPost(n) and
+    not any(ConsistencyConfiguration c).argHasPostUpdateExclude(n) and
+    msg = "ArgumentNode is missing PostUpdateNode."
+  }
+
+  // This predicate helps the compiler forget that in some languages
+  // it is impossible for a `PostUpdateNode` to be the target of
+  // `simpleLocalFlowStep`.
+  private predicate isPostUpdateNode(Node n) { n instanceof PostUpdateNode or none() }
+
+  query predicate postWithInFlow(Node n, string msg) {
+    isPostUpdateNode(n) and
+    not clearsContent(n, _) and
+    simpleLocalFlowStep(_, n) and
+    not any(ConsistencyConfiguration c).postWithInFlowExclude(n) and
+    msg = "PostUpdateNode should not be the target of local flow."
+  }
+
+  query predicate viableImplInCallContextTooLarge(
+    DataFlowCall call, DataFlowCall ctx, DataFlowCallable callable
+  ) {
+    callable = viableImplInCallContext(call, ctx) and
+    not callable = viableCallable(call) and
+    not any(ConsistencyConfiguration c).viableImplInCallContextTooLargeExclude(call, ctx, callable)
+  }
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplSpecific.qll

@@ -0,0 +1,11 @@
+/**
+ * Provides IR-specific definitions for use in the data flow library.
+ */
+module Private {
+  import DataFlowPrivate
+  import DataFlowDispatch
+}
+
+module Public {
+  import DataFlowUtil
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll

@@ -0,0 +1,560 @@
+private import cpp as Cpp
+private import DataFlowUtil
+private import semmle.code.cpp.ir.IR
+private import DataFlowDispatch
+private import DataFlowImplConsistency
+private import semmle.code.cpp.ir.internal.IRCppLanguage
+private import SsaInternals as Ssa
+
+/** Gets the callable in which this node occurs. */
+DataFlowCallable nodeGetEnclosingCallable(Node n) { result = n.getEnclosingCallable() }
+
+/** Holds if `p` is a `ParameterNode` of `c` with position `pos`. */
+predicate isParameterNode(ParameterNode p, DataFlowCallable c, ParameterPosition pos) {
+  p.isParameterOf(c, pos)
+}
+
+/** Holds if `arg` is an `ArgumentNode` of `c` with position `pos`. */
+predicate isArgumentNode(ArgumentNode arg, DataFlowCall c, ArgumentPosition pos) {
+  arg.argumentOf(c, pos)
+}
+
+/**
+ * A data flow node that occurs as the argument of a call and is passed as-is
+ * to the callable. Instance arguments (`this` pointer) and read side effects
+ * on parameters are also included.
+ */
+abstract class ArgumentNode extends Node {
+  /**
+   * Holds if this argument occurs at the given position in the given call.
+   * The instance argument is considered to have index `-1`.
+   */
+  abstract predicate argumentOf(DataFlowCall call, ArgumentPosition pos);
+
+  /** Gets the call in which this node is an argument. */
+  DataFlowCall getCall() { this.argumentOf(result, _) }
+}
+
+/**
+ * A data flow node that occurs as the argument to a call, or an
+ * implicit `this` pointer argument.
+ */
+private class PrimaryArgumentNode extends ArgumentNode, OperandNode {
+  override ArgumentOperand op;
+
+  PrimaryArgumentNode() { exists(CallInstruction call | op = call.getAnArgumentOperand()) }
+
+  override predicate argumentOf(DataFlowCall call, ArgumentPosition pos) {
+    op = call.getArgumentOperand(pos.(DirectPosition).getIndex())
+  }
+
+  override string toStringImpl() { result = argumentOperandToString(op) }
+}
+
+private string argumentOperandToString(ArgumentOperand op) {
+  exists(Expr unconverted |
+    unconverted = op.getDef().getUnconvertedResultExpression() and
+    result = unconverted.toString()
+  )
+  or
+  // Certain instructions don't map to an unconverted result expression. For these cases
+  // we fall back to a simpler naming scheme. This can happen in IR-generated constructors.
+  not exists(op.getDef().getUnconvertedResultExpression()) and
+  (
+    result = "Argument " + op.(PositionalArgumentOperand).getIndex()
+    or
+    op instanceof ThisArgumentOperand and result = "Argument this"
+  )
+}
+
+private class SideEffectArgumentNode extends ArgumentNode, SideEffectOperandNode {
+  override predicate argumentOf(DataFlowCall dfCall, ArgumentPosition pos) {
+    this.getCallInstruction() = dfCall and
+    pos.(IndirectionPosition).getArgumentIndex() = this.getArgumentIndex() and
+    pos.(IndirectionPosition).getIndirectionIndex() = super.getIndirectionIndex()
+  }
+
+  override string toStringImpl() {
+    result = argumentOperandToString(this.getAddressOperand()) + " indirection"
+  }
+}
+
+/** A parameter position represented by an integer. */
+class ParameterPosition = Position;
+
+/** An argument position represented by an integer. */
+class ArgumentPosition = Position;
+
+class Position extends TPosition {
+  abstract string toString();
+}
+
+class DirectPosition extends Position, TDirectPosition {
+  int index;
+
+  DirectPosition() { this = TDirectPosition(index) }
+
+  override string toString() { if index = -1 then result = "this" else result = index.toString() }
+
+  int getIndex() { result = index }
+}
+
+class IndirectionPosition extends Position, TIndirectionPosition {
+  int argumentIndex;
+  int indirectionIndex;
+
+  IndirectionPosition() { this = TIndirectionPosition(argumentIndex, indirectionIndex) }
+
+  override string toString() {
+    if argumentIndex = -1
+    then if indirectionIndex > 0 then result = "this indirection" else result = "this"
+    else
+      if indirectionIndex > 0
+      then result = argumentIndex.toString() + " indirection"
+      else result = argumentIndex.toString()
+  }
+
+  int getArgumentIndex() { result = argumentIndex }
+
+  int getIndirectionIndex() { result = indirectionIndex }
+}
+
+newtype TPosition =
+  TDirectPosition(int index) { exists(any(CallInstruction c).getArgument(index)) } or
+  TIndirectionPosition(int argumentIndex, int indirectionIndex) {
+    hasOperandAndIndex(_, any(CallInstruction call).getArgumentOperand(argumentIndex),
+      indirectionIndex)
+  }
+
+private newtype TReturnKind =
+  TNormalReturnKind(int index) {
+    exists(IndirectReturnNode return |
+      return.getAddressOperand() = any(ReturnValueInstruction r).getReturnAddressOperand() and
+      index = return.getIndirectionIndex() - 1 // We subtract one because the return loads the value.
+    )
+  } or
+  TIndirectReturnKind(int argumentIndex, int indirectionIndex) {
+    exists(IndirectReturnNode return, ReturnIndirectionInstruction returnInd |
+      returnInd.hasIndex(argumentIndex) and
+      return.getAddressOperand() = returnInd.getSourceAddressOperand() and
+      indirectionIndex = return.getIndirectionIndex()
+    )
+  }
+
+/**
+ * A return kind. A return kind describes how a value can be returned
+ * from a callable. For C++, this is simply a function return.
+ */
+class ReturnKind extends TReturnKind {
+  /** Gets a textual representation of this return kind. */
+  abstract string toString();
+}
+
+private class NormalReturnKind extends ReturnKind, TNormalReturnKind {
+  int index;
+
+  NormalReturnKind() { this = TNormalReturnKind(index) }
+
+  override string toString() { result = "indirect return" }
+}
+
+private class IndirectReturnKind extends ReturnKind, TIndirectReturnKind {
+  int argumentIndex;
+  int indirectionIndex;
+
+  IndirectReturnKind() { this = TIndirectReturnKind(argumentIndex, indirectionIndex) }
+
+  override string toString() { result = "indirect outparam[" + argumentIndex.toString() + "]" }
+}
+
+/** A data flow node that occurs as the result of a `ReturnStmt`. */
+class ReturnNode extends Node instanceof IndirectReturnNode {
+  /** Gets the kind of this returned value. */
+  abstract ReturnKind getKind();
+}
+
+/**
+ * This predicate represents an annoying hack that we have to do. We use the
+ * `ReturnIndirectionInstruction` to determine which variables need flow back
+ * out of a function. However, the IR will unconditionally create those for a
+ * variable passed to a function even though the variable was never updated by
+ * the function. And if a function has too many `ReturnNode`s the dataflow
+ * library lowers its precision for that function by disabling field flow.
+ *
+ * So we those eliminate `ReturnNode`s that would have otherwise been created
+ * by this unconditional `ReturnIndirectionInstruction` by requiring that there
+ * must exist an SSA definition of the IR variable in the function.
+ */
+private predicate hasNonInitializeParameterDef(IRVariable v) {
+  exists(Ssa::Def def |
+    not def.getDefiningInstruction() instanceof InitializeParameterInstruction and
+    v = def.getSourceVariable().getBaseVariable().(Ssa::BaseIRVariable).getIRVariable()
+  )
+}
+
+class ReturnIndirectionNode extends IndirectReturnNode, ReturnNode {
+  override ReturnKind getKind() {
+    exists(int argumentIndex, ReturnIndirectionInstruction returnInd |
+      returnInd.hasIndex(argumentIndex) and
+      this.getAddressOperand() = returnInd.getSourceAddressOperand() and
+      result = TIndirectReturnKind(argumentIndex, this.getIndirectionIndex()) and
+      hasNonInitializeParameterDef(returnInd.getIRVariable())
+    )
+    or
+    this.getAddressOperand() = any(ReturnValueInstruction r).getReturnAddressOperand() and
+    result = TNormalReturnKind(this.getIndirectionIndex() - 1)
+  }
+}
+
+private Operand fullyConvertedCallStep(Operand op) {
+  not exists(getANonConversionUse(op)) and
+  exists(Instruction instr |
+    conversionFlow(op, instr, _) and
+    result = getAUse(instr)
+  )
+}
+
+/**
+ * Gets the instruction that uses this operand, if the instruction is not
+ * ignored for dataflow purposes.
+ */
+private Instruction getUse(Operand op) {
+  result = op.getUse() and
+  not Ssa::ignoreOperand(op)
+}
+
+/** Gets a use of the instruction `instr` that is not ignored for dataflow purposes. */
+Operand getAUse(Instruction instr) {
+  result = instr.getAUse() and
+  not Ssa::ignoreOperand(result)
+}
+
+/**
+ * Gets a use of `operand` that is:
+ * - not ignored for dataflow purposes, and
+ * - not a conversion-like instruction.
+ */
+private Instruction getANonConversionUse(Operand operand) {
+  result = getUse(operand) and
+  not conversionFlow(_, result, _)
+}
+
+/**
+ * Gets the operand that represents the first use of the value of `call` following
+ * a sequence of conversion-like instructions.
+ */
+predicate operandForfullyConvertedCall(Operand operand, CallInstruction call) {
+  exists(getANonConversionUse(operand)) and
+  (
+    operand = getAUse(call)
+    or
+    operand = fullyConvertedCallStep*(getAUse(call))
+  )
+}
+
+/**
+ * Gets the instruction that represents the first use of the value of `call` following
+ * a sequence of conversion-like instructions.
+ *
+ * This predicate only holds if there is no suitable operand (i.e., no operand of a non-
+ * conversion instruction) to use to represent the value of `call` after conversions.
+ */
+predicate instructionForfullyConvertedCall(Instruction instr, CallInstruction call) {
+  not operandForfullyConvertedCall(_, call) and
+  (
+    // If there is no use of the call then we pick the call instruction
+    not exists(getAUse(call)) and
+    instr = call
+    or
+    // Otherwise, flow to the first non-conversion use.
+    exists(Operand operand | operand = fullyConvertedCallStep*(getAUse(call)) |
+      instr = getANonConversionUse(operand)
+    )
+  )
+}
+
+/** Holds if `node` represents the output node for `call`. */
+private predicate simpleOutNode(Node node, CallInstruction call) {
+  operandForfullyConvertedCall(node.asOperand(), call)
+  or
+  instructionForfullyConvertedCall(node.asInstruction(), call)
+}
+
+/** A data flow node that represents the output of a call. */
+class OutNode extends Node {
+  OutNode() {
+    // Return values not hidden behind indirections
+    simpleOutNode(this, _)
+    or
+    // Return values hidden behind indirections
+    this instanceof IndirectReturnOutNode
+    or
+    // Modified arguments hidden behind indirections
+    this instanceof IndirectArgumentOutNode
+  }
+
+  /** Gets the underlying call. */
+  abstract DataFlowCall getCall();
+
+  abstract ReturnKind getReturnKind();
+}
+
+private class DirectCallOutNode extends OutNode {
+  CallInstruction call;
+
+  DirectCallOutNode() { simpleOutNode(this, call) }
+
+  override DataFlowCall getCall() { result = call }
+
+  override ReturnKind getReturnKind() { result = TNormalReturnKind(0) }
+}
+
+private class IndirectCallOutNode extends OutNode, IndirectReturnOutNode {
+  override DataFlowCall getCall() { result = this.getCallInstruction() }
+
+  override ReturnKind getReturnKind() { result = TNormalReturnKind(this.getIndirectionIndex()) }
+}
+
+private class SideEffectOutNode extends OutNode, IndirectArgumentOutNode {
+  override DataFlowCall getCall() { result = this.getCallInstruction() }
+
+  override ReturnKind getReturnKind() {
+    result = TIndirectReturnKind(this.getArgumentIndex(), this.getIndirectionIndex())
+  }
+}
+
+/**
+ * Gets a node that can read the value returned from `call` with return kind
+ * `kind`.
+ */
+OutNode getAnOutNode(DataFlowCall call, ReturnKind kind) {
+  result.getCall() = call and
+  result.getReturnKind() = kind
+}
+
+/**
+ * Holds if data can flow from `node1` to `node2` in a way that loses the
+ * calling context. For example, this would happen with flow through a
+ * global or static variable.
+ */
+predicate jumpStep(Node n1, Node n2) {
+  exists(Cpp::GlobalOrNamespaceVariable v |
+    v =
+      n1.asInstruction()
+          .(StoreInstruction)
+          .getResultAddress()
+          .(VariableAddressInstruction)
+          .getAstVariable() and
+    v = n2.asVariable()
+    or
+    v =
+      n2.asInstruction()
+          .(LoadInstruction)
+          .getSourceAddress()
+          .(VariableAddressInstruction)
+          .getAstVariable() and
+    v = n1.asVariable()
+  )
+}
+
+/**
+ * Holds if data can flow from `node1` to `node2` via an assignment to `f`.
+ * Thus, `node2` references an object with a field `f` that contains the
+ * value of `node1`.
+ */
+predicate storeStep(Node node1, Content c, PostFieldUpdateNode node2) {
+  exists(int indirectionIndex1, int numberOfLoads, StoreInstruction store |
+    nodeHasInstruction(node1, store, pragma[only_bind_into](indirectionIndex1)) and
+    node2.getIndirectionIndex() = 1 and
+    numberOfLoadsFromOperand(node2.getFieldAddress(), store.getDestinationAddressOperand(),
+      numberOfLoads)
+  |
+    exists(FieldContent fc | fc = c |
+      fc.getField() = node2.getUpdatedField() and
+      fc.getIndirectionIndex() = 1 + indirectionIndex1 + numberOfLoads
+    )
+    or
+    exists(UnionContent uc | uc = c |
+      uc.getAField() = node2.getUpdatedField() and
+      uc.getIndirectionIndex() = 1 + indirectionIndex1 + numberOfLoads
+    )
+  )
+}
+
+/**
+ * Holds if `operandFrom` flows to `operandTo` using a sequence of conversion-like
+ * operations and exactly `n` `LoadInstruction` operations.
+ */
+private predicate numberOfLoadsFromOperandRec(Operand operandFrom, Operand operandTo, int ind) {
+  exists(LoadInstruction load | load.getSourceAddressOperand() = operandFrom |
+    operandTo = operandFrom and ind = 0
+    or
+    numberOfLoadsFromOperand(load.getAUse(), operandTo, ind - 1)
+  )
+  or
+  exists(Operand op, Instruction instr |
+    instr = op.getDef() and
+    conversionFlow(operandFrom, instr, _) and
+    numberOfLoadsFromOperand(op, operandTo, ind)
+  )
+}
+
+/**
+ * Holds if `operandFrom` flows to `operandTo` using a sequence of conversion-like
+ * operations and exactly `n` `LoadInstruction` operations.
+ */
+private predicate numberOfLoadsFromOperand(Operand operandFrom, Operand operandTo, int n) {
+  numberOfLoadsFromOperandRec(operandFrom, operandTo, n)
+  or
+  not any(LoadInstruction load).getSourceAddressOperand() = operandFrom and
+  not conversionFlow(operandFrom, _, _) and
+  operandFrom = operandTo and
+  n = 0
+}
+
+// Needed to join on both an operand and an index at the same time.
+pragma[noinline]
+predicate nodeHasOperand(Node node, Operand operand, int indirectionIndex) {
+  node.asOperand() = operand and indirectionIndex = 0
+  or
+  hasOperandAndIndex(node, operand, indirectionIndex)
+}
+
+// Needed to join on both an instruction and an index at the same time.
+pragma[noinline]
+predicate nodeHasInstruction(Node node, Instruction instr, int indirectionIndex) {
+  node.asInstruction() = instr and indirectionIndex = 0
+  or
+  hasInstructionAndIndex(node, instr, indirectionIndex)
+}
+
+/**
+ * Holds if data can flow from `node1` to `node2` via a read of `f`.
+ * Thus, `node1` references an object with a field `f` whose value ends up in
+ * `node2`.
+ */
+predicate readStep(Node node1, Content c, Node node2) {
+  exists(FieldAddress fa1, Operand operand, int numberOfLoads, int indirectionIndex2 |
+    nodeHasOperand(node2, operand, indirectionIndex2) and
+    nodeHasOperand(node1, fa1.getObjectAddressOperand(), _) and
+    numberOfLoadsFromOperand(fa1, operand, numberOfLoads)
+  |
+    exists(FieldContent fc | fc = c |
+      fc.getField() = fa1.getField() and
+      fc.getIndirectionIndex() = indirectionIndex2 + numberOfLoads
+    )
+    or
+    exists(UnionContent uc | uc = c |
+      uc.getAField() = fa1.getField() and
+      uc.getIndirectionIndex() = indirectionIndex2 + numberOfLoads
+    )
+  )
+}
+
+/**
+ * Holds if values stored inside content `c` are cleared at node `n`.
+ */
+predicate clearsContent(Node n, Content c) {
+  none() // stub implementation
+}
+
+/**
+ * Holds if the value that is being tracked is expected to be stored inside content `c`
+ * at node `n`.
+ */
+predicate expectsContent(Node n, ContentSet c) { none() }
+
+/** Gets the type of `n` used for type pruning. */
+DataFlowType getNodeType(Node n) {
+  suppressUnusedNode(n) and
+  result instanceof VoidType // stub implementation
+}
+
+/** Gets a string representation of a type returned by `getNodeType`. */
+string ppReprType(DataFlowType t) { none() } // stub implementation
+
+/**
+ * Holds if `t1` and `t2` are compatible, that is, whether data can flow from
+ * a node of type `t1` to a node of type `t2`.
+ */
+pragma[inline]
+predicate compatibleTypes(DataFlowType t1, DataFlowType t2) {
+  any() // stub implementation
+}
+
+private predicate suppressUnusedNode(Node n) { any() }
+
+//////////////////////////////////////////////////////////////////////////////
+// Java QL library compatibility wrappers
+//////////////////////////////////////////////////////////////////////////////
+/** A node that performs a type cast. */
+class CastNode extends Node {
+  CastNode() { none() } // stub implementation
+}
+
+/**
+ * A function that may contain code or a variable that may contain itself. When
+ * flow crosses from one _enclosing callable_ to another, the interprocedural
+ * data-flow library discards call contexts and inserts a node in the big-step
+ * relation used for human-readable path explanations.
+ */
+class DataFlowCallable = Cpp::Declaration;
+
+class DataFlowExpr = Expr;
+
+class DataFlowType = Type;
+
+/** A function call relevant for data flow. */
+class DataFlowCall extends CallInstruction {
+  Function getEnclosingCallable() { result = this.getEnclosingFunction() }
+}
+
+predicate isUnreachableInCall(Node n, DataFlowCall call) { none() } // stub implementation
+
+int accessPathLimit() { result = 5 }
+
+/**
+ * Holds if access paths with `c` at their head always should be tracked at high
+ * precision. This disables adaptive access path precision for such access paths.
+ */
+predicate forceHighPrecision(Content c) { none() }
+
+/** The unit type. */
+private newtype TUnit = TMkUnit()
+
+/** The trivial type with a single element. */
+class Unit extends TUnit {
+  /** Gets a textual representation of this element. */
+  string toString() { result = "unit" }
+}
+
+/** Holds if `n` should be hidden from path explanations. */
+predicate nodeIsHidden(Node n) { n instanceof OperandNode and not n instanceof ArgumentNode }
+
+class LambdaCallKind = Unit;
+
+/** Holds if `creation` is an expression that creates a lambda of kind `kind` for `c`. */
+predicate lambdaCreation(Node creation, LambdaCallKind kind, DataFlowCallable c) { none() }
+
+/** Holds if `call` is a lambda call of kind `kind` where `receiver` is the lambda expression. */
+predicate lambdaCall(DataFlowCall call, LambdaCallKind kind, Node receiver) { none() }
+
+/** Extra data-flow steps needed for lambda flow analysis. */
+predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preservesValue) { none() }
+
+/**
+ * Holds if flow is allowed to pass from parameter `p` and back to itself as a
+ * side-effect, resulting in a summary from `p` to itself.
+ *
+ * One example would be to allow flow like `p.foo = p.bar;`, which is disallowed
+ * by default as a heuristic.
+ */
+predicate allowParameterReturnInSelf(ParameterNode p) { none() }
+
+private class MyConsistencyConfiguration extends Consistency::ConsistencyConfiguration {
+  override predicate argHasPostUpdateExclude(ArgumentNode n) {
+    // The rules for whether an IR argument gets a post-update node are too
+    // complex to model here.
+    any()
+  }
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/ModelUtil.qll

@@ -0,0 +1,93 @@
+/**
+ * Provides predicates for mapping the `FunctionInput` and `FunctionOutput`
+ * classes used in function models to the corresponding instructions.
+ */
+
+private import semmle.code.cpp.ir.IR
+private import experimental.semmle.code.cpp.ir.dataflow.DataFlow
+private import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowUtil
+private import SsaInternals as Ssa
+
+/**
+ * Gets the instruction that goes into `input` for `call`.
+ */
+DataFlow::Node callInput(CallInstruction call, FunctionInput input) {
+  // An argument or qualifier
+  exists(int index |
+    result.asOperand() = call.getArgumentOperand(index) and
+    input.isParameterOrQualifierAddress(index)
+  )
+  or
+  // A value pointed to by an argument or qualifier
+  exists(int index, int indirectionIndex |
+    hasOperandAndIndex(result, call.getArgumentOperand(index), indirectionIndex) and
+    input.isParameterDerefOrQualifierObject(index, indirectionIndex)
+  )
+  or
+  exists(int ind |
+    result = getIndirectReturnOutNode(call, ind) and
+    input.isReturnValueDeref(ind)
+  )
+}
+
+/**
+ * Gets the instruction that holds the `output` for `call`.
+ */
+Node callOutput(CallInstruction call, FunctionOutput output) {
+  // The return value
+  result.asInstruction() = call and
+  output.isReturnValue()
+  or
+  // The side effect of a call on the value pointed to by an argument or qualifier
+  exists(int index, int indirectionIndex |
+    result.(IndirectArgumentOutNode).getArgumentIndex() = index and
+    result.(IndirectArgumentOutNode).getIndirectionIndex() = indirectionIndex and
+    result.(IndirectArgumentOutNode).getCallInstruction() = call and
+    output.isParameterDerefOrQualifierObject(index, indirectionIndex)
+  )
+  or
+  exists(int ind |
+    result = getIndirectReturnOutNode(call, ind) and
+    output.isReturnValueDeref(ind)
+  )
+}
+
+DataFlow::Node callInput(CallInstruction call, FunctionInput input, int d) {
+  exists(DataFlow::Node n | n = callInput(call, input) and d > 0 |
+    // An argument or qualifier
+    hasOperandAndIndex(result, n.asOperand(), d)
+    or
+    exists(Operand operand, int indirectionIndex |
+      // A value pointed to by an argument or qualifier
+      hasOperandAndIndex(n, operand, indirectionIndex) and
+      hasOperandAndIndex(result, operand, indirectionIndex + d)
+    )
+  )
+}
+
+private IndirectReturnOutNode getIndirectReturnOutNode(CallInstruction call, int d) {
+  result.getCallInstruction() = call and
+  result.getIndirectionIndex() = d
+}
+
+/**
+ * Gets the instruction that holds the `output` for `call`.
+ */
+bindingset[d]
+Node callOutput(CallInstruction call, FunctionOutput output, int d) {
+  exists(DataFlow::Node n | n = callOutput(call, output) and d > 0 |
+    // The return value
+    result = getIndirectReturnOutNode(n.asInstruction(), d)
+    or
+    // If there isn't an indirect out node for the call with indirection `d` then
+    // we conflate this with the underlying `CallInstruction`.
+    not exists(getIndirectReturnOutNode(call, d)) and
+    n.asInstruction() = result.asInstruction()
+    or
+    // The side effect of a call on the value pointed to by an argument or qualifier
+    exists(Operand operand, int indirectionIndex |
+      Ssa::outNodeHasAddressAndIndex(n, operand, indirectionIndex) and
+      Ssa::outNodeHasAddressAndIndex(result, operand, indirectionIndex + d)
+    )
+  )
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/PrintIRLocalFlow.qll

@@ -0,0 +1,136 @@
+private import cpp
+// The `ValueNumbering` library has to be imported right after `cpp` to ensure
+// that the cached IR gets the same checksum here as it does in queries that use
+// `ValueNumbering` without `DataFlow`.
+private import semmle.code.cpp.ir.ValueNumbering
+private import semmle.code.cpp.ir.IR
+private import semmle.code.cpp.ir.dataflow.DataFlow
+private import semmle.code.cpp.ir.dataflow.internal.DataFlowUtil
+private import PrintIRUtilities
+
+/**
+ * Gets the local dataflow from other nodes in the same function to this node.
+ */
+private string getFromFlow(DataFlow::Node useNode, int order1, int order2) {
+  exists(DataFlow::Node defNode, string prefix |
+    (
+      simpleLocalFlowStep(defNode, useNode) and prefix = ""
+      or
+      any(DataFlow::Configuration cfg).isAdditionalFlowStep(defNode, useNode) and
+      defNode.getEnclosingCallable() = useNode.getEnclosingCallable() and
+      prefix = "+"
+    ) and
+    if defNode.asInstruction() = useNode.asOperand().getAnyDef()
+    then
+      // Shorthand for flow from the def of this operand.
+      result = prefix + "def" and
+      order1 = -1 and
+      order2 = 0
+    else
+      if defNode.asOperand().getUse() = useNode.asInstruction()
+      then
+        // Shorthand for flow from an operand of this instruction
+        result = prefix + defNode.asOperand().getDumpId() and
+        order1 = -1 and
+        order2 = defNode.asOperand().getDumpSortOrder()
+      else result = prefix + nodeId(defNode, order1, order2)
+  )
+}
+
+/**
+ * Gets the local dataflow from this node to other nodes in the same function.
+ */
+private string getToFlow(DataFlow::Node defNode, int order1, int order2) {
+  exists(DataFlow::Node useNode, string prefix |
+    (
+      simpleLocalFlowStep(defNode, useNode) and prefix = ""
+      or
+      any(DataFlow::Configuration cfg).isAdditionalFlowStep(defNode, useNode) and
+      defNode.getEnclosingCallable() = useNode.getEnclosingCallable() and
+      prefix = "+"
+    ) and
+    if useNode.asInstruction() = defNode.asOperand().getUse()
+    then
+      // Shorthand for flow to this operand's instruction.
+      result = prefix + "result" and
+      order1 = -1 and
+      order2 = 0
+    else result = prefix + nodeId(useNode, order1, order2)
+  )
+}
+
+/**
+ * Gets the properties of the dataflow node `node`.
+ */
+private string getNodeProperty(DataFlow::Node node, string key) {
+  // List dataflow into and out of this node. Flow into this node is printed as `src->@`, and flow
+  // out of this node is printed as `@->dest`.
+  key = "flow" and
+  result =
+    strictconcat(string flow, boolean to, int order1, int order2 |
+      flow = getFromFlow(node, order1, order2) + "->@" and to = false
+      or
+      flow = "@->" + getToFlow(node, order1, order2) and to = true
+    |
+      flow, ", " order by to, order1, order2, flow
+    )
+  or
+  // Is this node a dataflow sink?
+  key = "sink" and
+  any(DataFlow::Configuration cfg).isSink(node) and
+  result = "true"
+  or
+  // Is this node a dataflow source?
+  key = "source" and
+  any(DataFlow::Configuration cfg).isSource(node) and
+  result = "true"
+  or
+  // Is this node a dataflow barrier, and if so, what kind?
+  key = "barrier" and
+  result =
+    strictconcat(string kind |
+      any(DataFlow::Configuration cfg).isBarrier(node) and kind = "full"
+      or
+      any(DataFlow::Configuration cfg).isBarrierIn(node) and kind = "in"
+      or
+      any(DataFlow::Configuration cfg).isBarrierOut(node) and kind = "out"
+    |
+      kind, ", "
+    )
+  or
+  // Is there partial flow from a source to this node?
+  // This property will only be emitted if partial flow is enabled by overriding
+  // `DataFlow::Configuration::explorationLimit()`.
+  key = "pflow" and
+  result =
+    strictconcat(DataFlow::PartialPathNode sourceNode, DataFlow::PartialPathNode destNode, int dist,
+      int order1, int order2 |
+      any(DataFlow::Configuration cfg).hasPartialFlow(sourceNode, destNode, dist) and
+      destNode.getNode() = node and
+      // Only print flow from a source in the same function.
+      sourceNode.getNode().getEnclosingCallable() = node.getEnclosingCallable()
+    |
+      nodeId(sourceNode.getNode(), order1, order2) + "+" + dist.toString(), ", "
+      order by
+        order1, order2, dist desc
+    )
+}
+
+/**
+ * Property provider for local IR dataflow.
+ */
+class LocalFlowPropertyProvider extends IRPropertyProvider {
+  override string getOperandProperty(Operand operand, string key) {
+    exists(DataFlow::Node node |
+      operand = node.asOperand() and
+      result = getNodeProperty(node, key)
+    )
+  }
+
+  override string getInstructionProperty(Instruction instruction, string key) {
+    exists(DataFlow::Node node |
+      instruction = node.asInstruction() and
+      result = getNodeProperty(node, key)
+    )
+  }
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/PrintIRStoreSteps.qll

@@ -0,0 +1,33 @@
+/**
+ * Print the dataflow local store steps in IR dumps.
+ */
+
+private import cpp
+// The `ValueNumbering` library has to be imported right after `cpp` to ensure
+// that the cached IR gets the same checksum here as it does in queries that use
+// `ValueNumbering` without `DataFlow`.
+private import semmle.code.cpp.ir.ValueNumbering
+private import semmle.code.cpp.ir.IR
+private import semmle.code.cpp.ir.dataflow.DataFlow
+private import semmle.code.cpp.ir.dataflow.internal.DataFlowUtil
+private import semmle.code.cpp.ir.dataflow.internal.DataFlowPrivate
+private import PrintIRUtilities
+
+/**
+ * Property provider for local IR dataflow store steps.
+ */
+class LocalFlowPropertyProvider extends IRPropertyProvider {
+  override string getInstructionProperty(Instruction instruction, string key) {
+    exists(DataFlow::Node objectNode, Content content |
+      key = "content[" + content.toString() + "]" and
+      instruction = objectNode.asInstruction() and
+      result =
+        strictconcat(string element, DataFlow::Node fieldNode |
+          storeStep(fieldNode, content, objectNode) and
+          element = nodeId(fieldNode, _, _)
+        |
+          element, ", "
+        )
+    )
+  }
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/PrintIRUtilities.qll

@@ -0,0 +1,39 @@
+/**
+ * Shared utilities used when printing dataflow annotations in IR dumps.
+ */
+
+private import cpp
+// The `ValueNumbering` library has to be imported right after `cpp` to ensure
+// that the cached IR gets the same checksum here as it does in queries that use
+// `ValueNumbering` without `DataFlow`.
+private import semmle.code.cpp.ir.ValueNumbering
+private import semmle.code.cpp.ir.IR
+private import semmle.code.cpp.ir.dataflow.DataFlow
+
+/**
+ * Gets a short ID for an IR dataflow node.
+ * - For `Instruction`s, this is just the result ID of the instruction (e.g. `m128`).
+ * - For `Operand`s, this is the label of the operand, prefixed with the result ID of the
+ *   instruction and a dot (e.g. `m128.left`).
+ * - For `Variable`s, this is the qualified name of the variable.
+ */
+string nodeId(DataFlow::Node node, int order1, int order2) {
+  exists(Instruction instruction | instruction = node.asInstruction() |
+    result = instruction.getResultId() and
+    order1 = instruction.getBlock().getDisplayIndex() and
+    order2 = instruction.getDisplayIndexInBlock()
+  )
+  or
+  exists(Operand operand, Instruction instruction |
+    operand = node.asOperand() and
+    instruction = operand.getUse()
+  |
+    result = instruction.getResultId() + "." + operand.getDumpId() and
+    order1 = instruction.getBlock().getDisplayIndex() and
+    order2 = instruction.getDisplayIndexInBlock()
+  )
+  or
+  result = "var(" + node.asVariable().getQualifiedName() + ")" and
+  order1 = 1000000 and
+  order2 = 0
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/SsaInternals.qll

@@ -0,0 +1,552 @@
+private import codeql.ssa.Ssa as SsaImplCommon
+private import semmle.code.cpp.ir.IR
+private import DataFlowUtil
+private import DataFlowImplCommon as DataFlowImplCommon
+private import semmle.code.cpp.models.interfaces.Allocation as Alloc
+private import semmle.code.cpp.models.interfaces.DataFlow as DataFlow
+private import semmle.code.cpp.ir.internal.IRCppLanguage
+private import DataFlowPrivate
+private import ssa0.SsaInternals as SsaInternals0
+import SsaInternalsCommon
+
+private module SourceVariables {
+  int getMaxIndirectionForIRVariable(IRVariable var) {
+    exists(Type type, boolean isGLValue |
+      var.getLanguageType().hasType(type, isGLValue) and
+      if isGLValue = true
+      then result = 1 + getMaxIndirectionsForType(type)
+      else result = getMaxIndirectionsForType(type)
+    )
+  }
+
+  class BaseSourceVariable = SsaInternals0::BaseSourceVariable;
+
+  class BaseIRVariable = SsaInternals0::BaseIRVariable;
+
+  class BaseCallVariable = SsaInternals0::BaseCallVariable;
+
+  cached
+  private newtype TSourceVariable =
+    TSourceIRVariable(BaseIRVariable baseVar, int ind) {
+      ind = [0 .. getMaxIndirectionForIRVariable(baseVar.getIRVariable())]
+    } or
+    TCallVariable(AllocationInstruction call, int ind) {
+      ind = [0 .. countIndirectionsForCppType(getResultLanguageType(call))]
+    }
+
+  abstract class SourceVariable extends TSourceVariable {
+    int ind;
+
+    bindingset[ind]
+    SourceVariable() { any() }
+
+    abstract string toString();
+
+    int getIndirection() { result = ind }
+
+    abstract BaseSourceVariable getBaseVariable();
+  }
+
+  class SourceIRVariable extends SourceVariable, TSourceIRVariable {
+    BaseIRVariable var;
+
+    SourceIRVariable() { this = TSourceIRVariable(var, ind) }
+
+    IRVariable getIRVariable() { result = var.getIRVariable() }
+
+    override BaseIRVariable getBaseVariable() { result.getIRVariable() = this.getIRVariable() }
+
+    override string toString() {
+      ind = 0 and
+      result = this.getIRVariable().toString()
+      or
+      ind > 0 and
+      result = this.getIRVariable().toString() + " indirection"
+    }
+  }
+
+  class CallVariable extends SourceVariable, TCallVariable {
+    AllocationInstruction call;
+
+    CallVariable() { this = TCallVariable(call, ind) }
+
+    AllocationInstruction getCall() { result = call }
+
+    override BaseCallVariable getBaseVariable() { result.getCallInstruction() = call }
+
+    override string toString() {
+      ind = 0 and
+      result = "Call"
+      or
+      ind > 0 and
+      result = "Call indirection"
+    }
+  }
+}
+
+import SourceVariables
+
+predicate hasIndirectOperand(Operand op, int indirectionIndex) {
+  exists(CppType type, int m |
+    not ignoreOperand(op) and
+    type = getLanguageType(op) and
+    m = countIndirectionsForCppType(type) and
+    indirectionIndex = [1 .. m]
+  )
+}
+
+predicate hasIndirectInstruction(Instruction instr, int indirectionIndex) {
+  exists(CppType type, int m |
+    not ignoreInstruction(instr) and
+    type = getResultLanguageType(instr) and
+    m = countIndirectionsForCppType(type) and
+    indirectionIndex = [1 .. m]
+  )
+}
+
+cached
+private newtype TDefOrUseImpl =
+  TDefImpl(Operand address, int indirectionIndex) {
+    isDef(_, _, address, _, _, indirectionIndex) and
+    // We only include the definition if the SSA pruning stage
+    // concluded that the definition is live after the write.
+    any(SsaInternals0::Def def).getAddressOperand() = address
+  } or
+  TUseImpl(Operand operand, int indirectionIndex) {
+    isUse(_, operand, _, _, indirectionIndex) and
+    not isDef(_, _, operand, _, _, _)
+  }
+
+abstract private class DefOrUseImpl extends TDefOrUseImpl {
+  /** Gets a textual representation of this element. */
+  abstract string toString();
+
+  /** Gets the block of this definition or use. */
+  abstract IRBlock getBlock();
+
+  /** Holds if this definition or use has index `index` in block `block`. */
+  abstract predicate hasIndexInBlock(IRBlock block, int index);
+
+  final predicate hasIndexInBlock(IRBlock block, int index, SourceVariable sv) {
+    this.hasIndexInBlock(block, index) and
+    sv = this.getSourceVariable()
+  }
+
+  /** Gets the location of this element. */
+  abstract Cpp::Location getLocation();
+
+  /**
+   * Gets the index (i.e., the number of loads required) of this
+   * definition or use.
+   *
+   * Note that this is _not_ the definition's (or use's) index in
+   * the enclosing basic block. To obtain this index, use
+   * `DefOrUseImpl::hasIndexInBlock/2` or `DefOrUseImpl::hasIndexInBlock/3`.
+   */
+  abstract int getIndirectionIndex();
+
+  /**
+   * Gets the instruction that computes the base of this definition or use.
+   * This is always a `VariableAddressInstruction` or an `AllocationInstruction`.
+   */
+  abstract Instruction getBase();
+
+  final BaseSourceVariable getBaseSourceVariable() {
+    exists(IRVariable var |
+      result.(BaseIRVariable).getIRVariable() = var and
+      instructionHasIRVariable(this.getBase(), var)
+    )
+    or
+    result.(BaseCallVariable).getCallInstruction() = this.getBase()
+  }
+
+  /** Gets the variable that is defined or used. */
+  final SourceVariable getSourceVariable() {
+    exists(BaseSourceVariable v, int ind |
+      sourceVariableHasBaseAndIndex(result, v, ind) and
+      defOrUseHasSourceVariable(this, v, ind)
+    )
+  }
+}
+
+pragma[noinline]
+private predicate instructionHasIRVariable(VariableAddressInstruction vai, IRVariable var) {
+  vai.getIRVariable() = var
+}
+
+private predicate defOrUseHasSourceVariable(DefOrUseImpl defOrUse, BaseSourceVariable bv, int ind) {
+  defHasSourceVariable(defOrUse, bv, ind)
+  or
+  useHasSourceVariable(defOrUse, bv, ind)
+}
+
+pragma[noinline]
+private predicate defHasSourceVariable(DefImpl def, BaseSourceVariable bv, int ind) {
+  bv = def.getBaseSourceVariable() and
+  ind = def.getIndirection()
+}
+
+pragma[noinline]
+private predicate useHasSourceVariable(UseImpl use, BaseSourceVariable bv, int ind) {
+  bv = use.getBaseSourceVariable() and
+  ind = use.getIndirection()
+}
+
+pragma[noinline]
+private predicate sourceVariableHasBaseAndIndex(SourceVariable v, BaseSourceVariable bv, int ind) {
+  v.getBaseVariable() = bv and
+  v.getIndirection() = ind
+}
+
+class DefImpl extends DefOrUseImpl, TDefImpl {
+  Operand address;
+  int ind;
+
+  DefImpl() { this = TDefImpl(address, ind) }
+
+  override Instruction getBase() { isDef(_, _, address, result, _, _) }
+
+  Operand getAddressOperand() { result = address }
+
+  int getIndirection() { isDef(_, _, address, _, result, ind) }
+
+  override int getIndirectionIndex() { result = ind }
+
+  Instruction getDefiningInstruction() { isDef(_, result, address, _, _, _) }
+
+  override string toString() { result = "DefImpl" }
+
+  override IRBlock getBlock() { result = this.getDefiningInstruction().getBlock() }
+
+  override Cpp::Location getLocation() { result = this.getDefiningInstruction().getLocation() }
+
+  final override predicate hasIndexInBlock(IRBlock block, int index) {
+    this.getDefiningInstruction() = block.getInstruction(index)
+  }
+
+  predicate isCertain() { isDef(true, _, address, _, _, ind) }
+}
+
+class UseImpl extends DefOrUseImpl, TUseImpl {
+  Operand operand;
+  int ind;
+
+  UseImpl() { this = TUseImpl(operand, ind) }
+
+  Operand getOperand() { result = operand }
+
+  override string toString() { result = "UseImpl" }
+
+  final override predicate hasIndexInBlock(IRBlock block, int index) {
+    operand.getUse() = block.getInstruction(index)
+  }
+
+  final override IRBlock getBlock() { result = operand.getUse().getBlock() }
+
+  final override Cpp::Location getLocation() { result = operand.getLocation() }
+
+  final int getIndirection() { isUse(_, operand, _, result, ind) }
+
+  override int getIndirectionIndex() { result = ind }
+
+  override Instruction getBase() { isUse(_, operand, result, _, ind) }
+
+  predicate isCertain() { isUse(true, operand, _, _, ind) }
+}
+
+/**
+ * Holds if `defOrUse1` is a definition which is first read by `use`,
+ * or if `defOrUse1` is a use and `use` is a next subsequent use.
+ *
+ * In both cases, `use` can either be an explicit use written in the
+ * source file, or it can be a phi node as computed by the SSA library.
+ */
+predicate adjacentDefRead(DefOrUse defOrUse1, UseOrPhi use) {
+  exists(IRBlock bb1, int i1, SourceVariable v |
+    defOrUse1.asDefOrUse().hasIndexInBlock(bb1, i1, v)
+  |
+    exists(IRBlock bb2, int i2 |
+      adjacentDefRead(_, pragma[only_bind_into](bb1), pragma[only_bind_into](i1),
+        pragma[only_bind_into](bb2), pragma[only_bind_into](i2))
+    |
+      use.asDefOrUse().(UseImpl).hasIndexInBlock(bb2, i2, v)
+    )
+    or
+    exists(PhiNode phi |
+      lastRefRedef(_, bb1, i1, phi) and
+      use.asPhi() = phi and
+      phi.getSourceVariable() = pragma[only_bind_into](v)
+    )
+  )
+}
+
+private predicate useToNode(UseOrPhi use, Node nodeTo) {
+  exists(UseImpl useImpl |
+    useImpl = use.asDefOrUse() and
+    nodeHasOperand(nodeTo, useImpl.getOperand(), useImpl.getIndirectionIndex())
+  )
+  or
+  nodeTo.(SsaPhiNode).getPhiNode() = use.asPhi()
+}
+
+pragma[noinline]
+predicate outNodeHasAddressAndIndex(
+  IndirectArgumentOutNode out, Operand address, int indirectionIndex
+) {
+  out.getAddressOperand() = address and
+  out.getIndirectionIndex() = indirectionIndex
+}
+
+private predicate defToNode(Node nodeFrom, Def def) {
+  nodeHasInstruction(nodeFrom, def.getDefiningInstruction(), def.getIndirectionIndex())
+}
+
+/**
+ * INTERNAL: Do not use.
+ *
+ * Holds if `nodeFrom` is the node that correspond to the definition or use `defOrUse`.
+ */
+predicate nodeToDefOrUse(Node nodeFrom, SsaDefOrUse defOrUse) {
+  // Node -> Def
+  defToNode(nodeFrom, defOrUse)
+  or
+  // Node -> Use
+  useToNode(defOrUse, nodeFrom)
+}
+
+/**
+ * Perform a single conversion-like step from `nFrom` to `nTo`. This relation
+ * only holds when there is no use-use relation out of `nTo`.
+ */
+private predicate indirectConversionFlowStep(Node nFrom, Node nTo) {
+  not exists(UseOrPhi defOrUse |
+    nodeToDefOrUse(nTo, defOrUse) and
+    adjacentDefRead(defOrUse, _)
+  ) and
+  exists(Operand op1, Operand op2, int indirectionIndex, Instruction instr |
+    hasOperandAndIndex(nFrom, op1, pragma[only_bind_into](indirectionIndex)) and
+    hasOperandAndIndex(nTo, op2, pragma[only_bind_into](indirectionIndex)) and
+    instr = op2.getDef() and
+    conversionFlow(op1, instr, _)
+  )
+}
+
+/**
+ * The reason for this predicate is a bit annoying:
+ * We cannot mark a `PointerArithmeticInstruction` that computes an offset based on some SSA
+ * variable `x` as a use of `x` since this creates taint-flow in the following example:
+ * ```c
+ * int x = array[source]
+ * sink(*array)
+ * ```
+ * This is because `source` would flow from the operand of `PointerArithmeticInstruction` to the
+ * result of the instruction, and into the `IndirectOperand` that represents the value of `*array`.
+ * Then, via use-use flow, flow will arrive at `*array` in `sink(*array)`.
+ *
+ * So this predicate recurses back along conversions and `PointerArithmeticInstruction`s to find the
+ * first use that has provides use-use flow, and uses that target as the target of the `nodeFrom`.
+ */
+private predicate adjustForPointerArith(Node nodeFrom, UseOrPhi use) {
+  nodeFrom = any(PostUpdateNode pun).getPreUpdateNode() and
+  exists(DefOrUse defOrUse, Node adjusted |
+    indirectConversionFlowStep*(adjusted, nodeFrom) and
+    nodeToDefOrUse(adjusted, defOrUse) and
+    adjacentDefRead(defOrUse, use)
+  )
+}
+
+/** Holds if there is def-use or use-use flow from `nodeFrom` to `nodeTo`. */
+predicate ssaFlow(Node nodeFrom, Node nodeTo) {
+  // `nodeFrom = any(PostUpdateNode pun).getPreUpdateNode()` is implied by adjustedForPointerArith.
+  exists(UseOrPhi use |
+    adjustForPointerArith(nodeFrom, use) and
+    useToNode(use, nodeTo)
+  )
+  or
+  not nodeFrom = any(PostUpdateNode pun).getPreUpdateNode() and
+  exists(DefOrUse defOrUse1, UseOrPhi use |
+    nodeToDefOrUse(nodeFrom, defOrUse1) and
+    adjacentDefRead(defOrUse1, use) and
+    useToNode(use, nodeTo)
+  )
+}
+
+/** Holds if `nodeTo` receives flow from the phi node `nodeFrom`. */
+predicate fromPhiNode(SsaPhiNode nodeFrom, Node nodeTo) {
+  exists(PhiNode phi, SourceVariable sv, IRBlock bb1, int i1, UseOrPhi use |
+    phi = nodeFrom.getPhiNode() and
+    phi.definesAt(sv, bb1, i1) and
+    useToNode(use, nodeTo)
+  |
+    exists(IRBlock bb2, int i2 |
+      use.asDefOrUse().hasIndexInBlock(bb2, i2, sv) and
+      adjacentDefRead(phi, bb1, i1, bb2, i2)
+    )
+    or
+    exists(PhiNode phiTo |
+      lastRefRedef(phi, _, _, phiTo) and
+      nodeTo.(SsaPhiNode).getPhiNode() = phiTo
+    )
+  )
+}
+
+private SsaInternals0::SourceVariable getOldSourceVariable(SourceVariable v) {
+  v.getBaseVariable().(BaseIRVariable).getIRVariable() =
+    result.getBaseVariable().(SsaInternals0::BaseIRVariable).getIRVariable()
+  or
+  v.getBaseVariable().(BaseCallVariable).getCallInstruction() =
+    result.getBaseVariable().(SsaInternals0::BaseCallVariable).getCallInstruction()
+}
+
+/**
+ * Holds if there is a write at index `i` in basic block `bb` to variable `v` that's
+ * subsequently read (as determined by the SSA pruning stage).
+ */
+private predicate variableWriteCand(IRBlock bb, int i, SourceVariable v) {
+  exists(SsaInternals0::Def def, SsaInternals0::SourceVariable v0 |
+    def.asDefOrUse().hasIndexInBlock(bb, i, v0) and
+    v0 = getOldSourceVariable(v)
+  )
+}
+
+private module SsaInput implements SsaImplCommon::InputSig {
+  import InputSigCommon
+  import SourceVariables
+
+  /**
+   * Holds if the `i`'th write in block `bb` writes to the variable `v`.
+   * `certain` is `true` if the write is guaranteed to overwrite the entire variable.
+   */
+  predicate variableWrite(IRBlock bb, int i, SourceVariable v, boolean certain) {
+    DataFlowImplCommon::forceCachingInSameStage() and
+    variableWriteCand(bb, i, v) and
+    exists(DefImpl def | def.hasIndexInBlock(bb, i, v) |
+      if def.isCertain() then certain = true else certain = false
+    )
+  }
+
+  /**
+   * Holds if the `i`'th read in block `bb` reads to the variable `v`.
+   * `certain` is `true` if the read is guaranteed. For C++, this is always the case.
+   */
+  predicate variableRead(IRBlock bb, int i, SourceVariable v, boolean certain) {
+    exists(UseImpl use | use.hasIndexInBlock(bb, i, v) |
+      if use.isCertain() then certain = true else certain = false
+    )
+  }
+}
+
+/**
+ * The final SSA predicates used for dataflow purposes.
+ */
+cached
+module SsaCached {
+  /**
+   * Holds if `def` is accessed at index `i1` in basic block `bb1` (either a read
+   * or a write), `def` is read at index `i2` in basic block `bb2`, and there is a
+   * path between them without any read of `def`.
+   */
+  cached
+  predicate adjacentDefRead(Definition def, IRBlock bb1, int i1, IRBlock bb2, int i2) {
+    SsaImpl::adjacentDefRead(def, bb1, i1, bb2, i2)
+  }
+
+  /**
+   * Holds if the node at index `i` in `bb` is a last reference to SSA definition
+   * `def`. The reference is last because it can reach another write `next`,
+   * without passing through another read or write.
+   */
+  cached
+  predicate lastRefRedef(Definition def, IRBlock bb, int i, Definition next) {
+    SsaImpl::lastRefRedef(def, bb, i, next)
+  }
+}
+
+cached
+private newtype TSsaDefOrUse =
+  TDefOrUse(DefOrUseImpl defOrUse) {
+    defOrUse instanceof UseImpl
+    or
+    // Like in the pruning stage, we only include definition that's live after the
+    // write as the final definitions computed by SSA.
+    exists(Definition def, SourceVariable sv, IRBlock bb, int i |
+      def.definesAt(sv, bb, i) and
+      defOrUse.(DefImpl).hasIndexInBlock(bb, i, sv)
+    )
+  } or
+  TPhi(PhiNode phi)
+
+abstract private class SsaDefOrUse extends TSsaDefOrUse {
+  string toString() { none() }
+
+  DefOrUseImpl asDefOrUse() { none() }
+
+  PhiNode asPhi() { none() }
+
+  abstract Location getLocation();
+}
+
+class DefOrUse extends TDefOrUse, SsaDefOrUse {
+  DefOrUseImpl defOrUse;
+
+  DefOrUse() { this = TDefOrUse(defOrUse) }
+
+  final override DefOrUseImpl asDefOrUse() { result = defOrUse }
+
+  final override Location getLocation() { result = defOrUse.getLocation() }
+
+  final SourceVariable getSourceVariable() { result = defOrUse.getSourceVariable() }
+
+  override string toString() { result = defOrUse.toString() }
+}
+
+class Phi extends TPhi, SsaDefOrUse {
+  PhiNode phi;
+
+  Phi() { this = TPhi(phi) }
+
+  final override PhiNode asPhi() { result = phi }
+
+  final override Location getLocation() { result = phi.getBasicBlock().getLocation() }
+
+  override string toString() { result = "Phi" }
+}
+
+class UseOrPhi extends SsaDefOrUse {
+  UseOrPhi() {
+    this.asDefOrUse() instanceof UseImpl
+    or
+    this instanceof Phi
+  }
+
+  final override Location getLocation() {
+    result = this.asDefOrUse().getLocation() or result = this.(Phi).getLocation()
+  }
+}
+
+class Def extends DefOrUse {
+  override DefImpl defOrUse;
+
+  Operand getAddressOperand() { result = defOrUse.getAddressOperand() }
+
+  Instruction getAddress() { result = this.getAddressOperand().getDef() }
+
+  /**
+   * This predicate ensures that joins go from `defOrUse` to the result
+   * instead of the other way around.
+   */
+  pragma[inline]
+  int getIndirectionIndex() {
+    pragma[only_bind_into](result) = pragma[only_bind_out](defOrUse).getIndirectionIndex()
+  }
+
+  Instruction getDefiningInstruction() { result = defOrUse.getDefiningInstruction() }
+}
+
+private module SsaImpl = SsaImplCommon::Make<SsaInput>;
+
+class PhiNode = SsaImpl::PhiNode;
+
+class Definition = SsaImpl::Definition;
+
+import SsaCached