wip

.bazelrc

@@ -1,9 +1,3 @@
-common --enable_platform_specific_config
-
-build --repo_env=CC=clang --repo_env=CXX=clang++
-
-build:linux --cxxopt=-std=c++20
-build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
-build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
+build --repo_env=CC=clang --repo_env=CXX=clang++ --cxxopt="-std=c++17"
 
 try-import %workspace%/local.bazelrc

.bazelversion

@@ -1 +1 @@
-6.3.1
+5.0.0

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "extensions": [
-    "rust-lang.rust-analyzer",
+    "rust-lang.rust",
     "bungcip.better-toml",
     "github.vscode-codeql",
     "hbenl.vscode-test-explorer",

.git-blame-ignore-revs

@@ -1,21 +0,0 @@
-# .git-blame-ignore-revs
-# Auto-formatted Java
-730eae952139209fe9fdf598541d608f4c0c0c84
-# Auto-formatted C#
-5ad7ed49dd3de03ec6dcfcb6848758a6a987e11c
-# Auto-formatted C/C++
-ef97e539ec1971494d4bba5cafe82e00bc8217ac
-# Auto-formatted Python
-21d5fa836b3a7d020ba45e8b8168b145a9772131
-# Auto-formatted JavaScript
-8d97fe9ed327a9546ff2eaf515cf0f5214deddd9
-# Auto-formatted Ruby
-a5d229903d2f12d45f2c2c38822f1d0e7504ae7f
-# Auto-formatted Go
-08c658e66bf867090033ea096e244a93d46c0aa7
-# Auto-formatted Swift
-711d7057f79fb7d72fc3b35e010bd018f9009169
-# Auto-formatted shared ql packs
-3640b6d3a8ce9edf8e1d3ed106fe8526cf255bc0
-# Auto-formatted taint tracking files
-159d8e978c51959b380838c080d891b66e763b19

.github/actions/cache-query-compilation/action.yml

@@ -9,7 +9,7 @@ inputs:
 outputs:
   cache-dir:
     description: "The directory where the cache was stored"
-    value: ${{ steps.output-compilation-dir.outputs.compdir }}
+    value: ${{ steps.fill-compilation-dir.outputs.compdir }}
 
 runs:
   using: composite
@@ -27,9 +27,7 @@ runs:
       if: ${{ github.event_name == 'pull_request' }}
       uses: actions/cache/restore@v3
       with:
-        path: |
-          **/.cache
-          ~/.codeql/compile-cache
+        path: '**/.cache'
         key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }}
         restore-keys: |
           codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
@@ -39,111 +37,18 @@ runs:
       if: ${{ github.event_name != 'pull_request' }}
       uses: actions/cache@v3
       with:
-        path: |
-          **/.cache
-          ~/.codeql/compile-cache
+        path: '**/.cache'
         key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
         restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
           codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
           codeql-compile-${{ inputs.key }}-main-
-    - name: Output-compilationdir
-      id: output-compilation-dir
+    - name: Fill compilation cache directory
+      id: fill-compilation-dir
       shell: bash
       run: |
+        # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+        node $GITHUB_WORKSPACE/.github/actions/cache-query-compilation/move-caches.js ${COMBINED_CACHE_DIR}
+
         echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
       env:
         COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
-    - name: Fill compilation cache directory
-      id: fill-compilation-dir
-      uses: actions/github-script@v6
-      env: 
-        COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
-      with:
-        script: |
-          // # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-          // mkdir -p ${COMBINED_CACHE_DIR}
-          // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-          // # copy the contents of the .cache folders into the combined cache folder.
-          // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-          // # clean up the .cache folders
-          // rm -rf **/.cache/*
-
-          const fs = require("fs");
-          const path = require("path");
-          const os = require("os");
-
-          // the first argv is the cache folder to create.
-          const COMBINED_CACHE_DIR = process.env.COMBINED_CACHE_DIR;
-
-          function* walkCaches(dir) {
-            const files = fs.readdirSync(dir, { withFileTypes: true });
-            for (const file of files) {
-              if (file.isDirectory()) {
-                const filePath = path.join(dir, file.name);
-                yield* walkCaches(filePath);
-                if (file.name === ".cache") {
-                  yield filePath;
-                }
-              }
-            }
-          }
-
-          async function copyDir(src, dest) {
-            for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
-              const srcPath = path.join(src, file.name);
-              const destPath = path.join(dest, file.name);
-              if (file.isDirectory()) {
-                if (!fs.existsSync(destPath)) {
-                  fs.mkdirSync(destPath);
-                }
-                await copyDir(srcPath, destPath);
-              } else {
-                await fs.promises.copyFile(srcPath, destPath);
-              }
-            }
-          }
-
-          async function main() {
-            const cacheDirs = [...walkCaches(".")];
-
-            for (const dir of cacheDirs) {
-              console.log(`Found .cache dir at ${dir}`);
-            }
-
-            const globalCacheDir = path.join(os.homedir(), ".codeql", "compile-cache");
-            if (fs.existsSync(globalCacheDir)) {
-              console.log("Found global home dir: " + globalCacheDir);
-              cacheDirs.push(globalCacheDir);
-            }
-
-            if (cacheDirs.length === 0) {
-              console.log("No cache dirs found");
-              return;
-            }
-
-            // mkdir -p ${COMBINED_CACHE_DIR}
-            fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
-
-            // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-            await Promise.all(
-              cacheDirs.map((cacheDir) =>
-                (async function () {
-                  await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
-                  await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
-                })()
-              )
-            );
-
-            // # copy the contents of the .cache folders into the combined cache folder.
-            // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-            await Promise.all(
-              cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
-            );
-
-            // # clean up the .cache folders
-            // rm -rf **/.cache/*
-            await Promise.all(
-              cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
-            );
-          }
-          main();

.github/actions/cache-query-compilation/move-caches.js

@@ -0,0 +1,75 @@
+// # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+// mkdir -p ${COMBINED_CACHE_DIR}
+// rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+// # copy the contents of the .cache folders into the combined cache folder.
+// cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+// # clean up the .cache folders
+// rm -rf **/.cache/*
+
+const fs = require("fs");
+const path = require("path");
+
+// the first argv is the cache folder to create.
+const COMBINED_CACHE_DIR = process.argv[2];
+
+function* walkCaches(dir) {
+  const files = fs.readdirSync(dir, { withFileTypes: true });
+  for (const file of files) {
+    if (file.isDirectory()) {
+      const filePath = path.join(dir, file.name);
+      yield* walkCaches(filePath);
+      if (file.name === ".cache") {
+        yield filePath;
+      }
+    }
+  }
+}
+
+async function copyDir(src, dest) {
+  for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
+    const srcPath = path.join(src, file.name);
+    const destPath = path.join(dest, file.name);
+    if (file.isDirectory()) {
+      if (!fs.existsSync(destPath)) {
+        fs.mkdirSync(destPath);
+      }
+      await copyDir(srcPath, destPath);
+    } else {
+      await fs.promises.copyFile(srcPath, destPath);
+    }
+  }
+}
+
+async function main() {
+  const cacheDirs = [...walkCaches(".")];
+
+  for (const dir of cacheDirs) {
+    console.log(`Found .cache dir at ${dir}`);
+  }
+
+  // mkdir -p ${COMBINED_CACHE_DIR}
+  fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
+
+  // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+  await Promise.all(
+    cacheDirs.map((cacheDir) =>
+      (async function () {
+        await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
+        await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
+      })()
+    )
+  );
+
+  // # copy the contents of the .cache folders into the combined cache folder.
+  // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+  await Promise.all(
+    cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
+  );
+
+  // # clean up the .cache folders
+  // rm -rf **/.cache/*
+  await Promise.all(
+    cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
+  );
+}
+main();

.github/actions/find-latest-bundle/action.yml

@@ -0,0 +1,26 @@
+name: Find Latest CodeQL Bundle
+description: Finds the URL of the latest released version of the CodeQL bundle.
+outputs:
+  url:
+    description: The download URL of the latest CodeQL bundle release
+    value: ${{ steps.find-latest.outputs.url }}
+runs:
+  using: composite
+  steps:
+    - name: Find Latest Release
+      id: find-latest
+      shell: pwsh
+      run: |
+        $Latest = gh release list --repo github/codeql-action --exclude-drafts --limit 1000 |
+          ForEach-Object { $C = $_ -split "`t"; return @{ type = $C[1]; tag = $C[2]; } } |
+          Where-Object { $_.type -eq 'Latest' }
+
+        $Tag = $Latest.tag
+        if ($Tag -eq '') {
+          throw 'Failed to find latest bundle release.'
+        }
+
+        Write-Output "Latest bundle tag is '${Tag}'."
+        "url=https://github.com/github/codeql-action/releases/download/${Tag}/codeql-bundle-linux64.tar.gz" >> $env:GITHUB_OUTPUT
+      env:
+        GITHUB_TOKEN: ${{ github.token }}

.github/dependabot.yml

@@ -5,11 +5,6 @@ updates:
     schedule:
       interval: "daily"
 
-  - package-ecosystem: "cargo"
-    directory: "ql"
-    schedule:
-      interval: "daily"
-
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:

.github/labeler.yml

@@ -11,7 +11,7 @@ Go:
   - change-notes/**/*go.*
 
 Java:
-  - any: [ 'java/**/*', '!java/kotlin-extractor/**/*', '!java/ql/test/kotlin/**/*' ]
+  - any: [ 'java/**/*', '!java/kotlin-extractor/**/*', '!java/kotlin-explorer/**/*', '!java/ql/test/kotlin/**/*' ]
   - change-notes/**/*java.*
 
 JS:
@@ -20,6 +20,7 @@ JS:
 
 Kotlin:
   - java/kotlin-extractor/**/*
+  - java/kotlin-explorer/**/*
   - java/ql/test/kotlin/**/*
 
 Python:
@@ -45,7 +46,11 @@ documentation:
 
 # Since these are all shared files that need to be synced, just pick _one_ copy of each.
 "DataFlow Library":
-  - "shared/dataflow/**/*"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"
 
 "ATM":
   - javascript/ql/experimental/adaptivethreatmodeling/**/*

.github/workflows/atm-check-query-suite.yml

@@ -0,0 +1,102 @@
+name: "ATM - Check query suite"
+
+env:
+  QUERY_PACK: javascript/ql/experimental/adaptivethreatmodeling/src
+  QUERY_SUITE: codeql-suites/javascript-atm-code-scanning.qls
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/atm-check-query-suite.yml"
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+  workflow_dispatch:
+
+jobs:
+  atm-check-query-suite:
+    runs-on: ubuntu-latest-xl
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup CodeQL
+        uses: ./.github/actions/fetch-codeql
+        with:
+          channel: release
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: atm-suite
+
+      - name: Install ATM model
+        run: |
+          set -exu
+
+          # Install dependencies of ATM query pack, i.e. the ATM model
+          codeql pack install "${QUERY_PACK}"
+
+          # Retrieve model checksum
+          model_checksum=$(codeql resolve extensions "${QUERY_PACK}/${QUERY_SUITE}" | jq -r '.models[0].checksum')
+
+          # Trust the model so that we can use it in the ATM boosted queries
+          mkdir -p "$HOME/.config/codeql"
+          echo "--insecurely-execute-ml-model-checksums ${model_checksum}" >> "$HOME/.config/codeql/config"
+
+      - name: Create test DB
+        run: |
+          DB_PATH="${RUNNER_TEMP}/db"
+          echo "DB_PATH=${DB_PATH}" >> "${GITHUB_ENV}"
+
+          codeql database create "${DB_PATH}" --source-root config/atm --language javascript 
+
+      - name: Run ATM query suite
+        run: |
+          SARIF_PATH="${RUNNER_TEMP}/sarif.json"
+          echo "SARIF_PATH=${SARIF_PATH}" >> "${GITHUB_ENV}"
+
+          codeql database analyze \
+            --threads=0 \
+            --ram 50000 \
+            --format sarif-latest \
+            --output "${SARIF_PATH}" \
+            --sarif-group-rules-by-pack \
+            -vv \
+            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
+            -- \
+            "${DB_PATH}" \
+            "${QUERY_PACK}/${QUERY_SUITE}"
+
+      - name: Upload SARIF
+        uses: actions/upload-artifact@v3
+        with:
+          name: javascript-ml-powered-queries.sarif
+          path: "${{ env.SARIF_PATH }}"
+          retention-days: 5
+
+      - name: Check results
+        run: |
+          # We should run at least the ML-powered queries in `expected_rules`.
+          expected_rules="js/ml-powered/nosql-injection js/ml-powered/path-injection js/ml-powered/sql-injection js/ml-powered/xss"
+
+          for rule in ${expected_rules}; do
+            found_rule=$(jq --arg rule "${rule}" '[.runs[0].tool.extensions[].rules | select(. != null) |
+              flatten | .[].id] | any(. == $rule)' "${SARIF_PATH}")
+            if [[ "${found_rule}" != "true" ]]; then
+              echo "Expected SARIF output to contain rule '${rule}', but found no such rule."
+              exit 1
+            else
+              echo "Found rule '${rule}'."
+            fi
+          done
+
+          # We should have at least one alert from an ML-powered query.
+          num_alerts=$(jq '[.runs[0].results[] |
+            select(.properties.score != null and (.rule.id | startswith("js/ml-powered/")))] | length' \
+            "${SARIF_PATH}")
+          if [[ "${num_alerts}" -eq 0 ]]; then
+            echo "Expected to find at least one alert from an ML-powered query but found ${num_alerts}."
+            exit 1
+          else
+            echo "Found ${num_alerts} alerts from ML-powered queries.";
+          fi

.github/workflows/atm-model-integration-tests.yml

@@ -0,0 +1,12 @@
+name: ATM Model Integration Tests
+
+on:
+  workflow_dispatch:
+
+jobs:
+  hello-world:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: foo
+        run: echo "Hello world"

.github/workflows/check-change-note.yml

@@ -8,9 +8,9 @@ on:
       - "*/ql/src/**/*.qll"
       - "*/ql/lib/**/*.ql"
       - "*/ql/lib/**/*.qll"
-      - "*/ql/lib/**/*.yml"
       - "!**/experimental/**"
       - "!ql/**"
+      - "!swift/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:
@@ -26,9 +26,9 @@ jobs:
         run: |
           gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' |
           grep true -c
-      - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md', 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text, or released/x.y.z.md for released change-notes
+      - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md' or 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text.
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$") or test("/change-notes/released/[0-9]*[.][0-9]*[.][0-9]*[.]md$"))' |
+          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$"))' |
           grep true -c

.github/workflows/check-implicit-this.yml

@@ -1,29 +0,0 @@
-name: "Check implicit this warnings"
-
-on:
-  workflow_dispatch:
-  pull_request:
-    paths:
-      - "**qlpack.yml"
-    branches:
-      - main
-      - "rc/*"
-
-jobs:
-  check:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Check that implicit this warnings is enabled for all packs
-        shell: bash
-        run: |
-          EXIT_CODE=0
-          packs="$(find . -iname 'qlpack.yml')"
-          for pack_file in ${packs}; do
-            option="$(yq '.warnOnImplicitThis' ${pack_file})"
-            if [ "${option}" != "true" ]; then
-              echo "::error file=${pack_file}::warnOnImplicitThis property must be set to 'true' for pack ${pack_file}"
-              EXIT_CODE=1
-            fi
-          done
-          exit "${EXIT_CODE}"

.github/workflows/check-qldoc.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 2
 
@@ -26,8 +26,9 @@ jobs:
         shell: bash
         run: |
           EXIT_CODE=0
+          # TODO: remove the swift exception from the regex when we fix generated QLdoc
           # TODO: remove the shared exception from the regex when coverage of qlpacks without dbschemes is supported
-          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!(shared))[a-z]*/ql/lib' || true; } | sort -u)"
+          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!(swift|shared))[a-z]*/ql/lib' || true; } | sort -u)"
           for pack_dir in ${changed_lib_packs}; do
             lang="${pack_dir%/ql/lib}"
             codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"

.github/workflows/check-query-ids.yml

@@ -16,6 +16,6 @@ jobs:
     name: Check query IDs
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Check for duplicate query IDs
         run: python3 misc/scripts/check-query-ids.py

.github/workflows/close-stale.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/stale@v8
+    - uses: actions/stale@v7
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Comment or remove the `Stale` label in order to avoid having this issue closed in 7 days.'

.github/workflows/codeql-analysis.yml

@@ -28,12 +28,12 @@ jobs:
 
     steps:
     - name: Setup dotnet
-      uses: actions/setup-dotnet@v3
+      uses: actions/setup-dotnet@v2
       with:
-        dotnet-version: 7.0.102
+        dotnet-version: 6.0.202
 
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/compile-queries.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest-xl
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
         with:
@@ -24,14 +24,14 @@ jobs:
         with: 
           key: all-queries
       - name: check formatting
-        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 -n 3000 -P 10 codeql query format -q --check-only
+        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
       - name: compile queries - check-only
         # run with --check-only if running in a PR (github.sha != main)
         if : ${{ github.event_name == 'pull_request' }}
         shell: bash
-        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
       - name: compile queries - full
         # do full compile if running on main - this populates the cache
         if : ${{ github.event_name != 'pull_request' }}
         shell: bash
-        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/csharp-qltest.yml

@@ -29,7 +29,7 @@ jobs:
   qlupgrade:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - name: Check DB upgrade scripts
         run: |
@@ -52,7 +52,8 @@ jobs:
       matrix:
         slice: ["1/2", "2/2"]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
       - uses: ./csharp/actions/create-extractor-pack
       - name: Cache compilation cache
         id: query-cache
@@ -61,41 +62,25 @@ jobs:
           key: csharp-qltest-${{ matrix.slice }}
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          CODEQL_PATH=$(gh codeql version --format=json | jq -r .unpackedLocation)
+          # The legacy ASP extractor is not in this repo, so take the one from the nightly build
+          mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
+          # Safe guard against using the bundled extractor
+          rm -rf "$CODEQL_PATH/csharp"
+          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}
   unit-tests:
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-2019]
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Setup dotnet
         uses: actions/setup-dotnet@v3
         with:
-          dotnet-version: 7.0.102
+          dotnet-version: 6.0.202
       - name: Extractor unit tests
         run: |
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 extractor/Semmle.Util.Tests
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 extractor/Semmle.Extraction.Tests
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 autobuilder/Semmle.Autobuild.CSharp.Tests
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"
-        shell: bash
-  stubgentest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./csharp/actions/create-extractor-pack
-      - name: Run stub generator tests
-        run: |
-          # Generate (Asp)NetCore stubs
-          STUBS_PATH=stubs_output
-          python3 ql/src/Stubs/make_stubs_nuget.py webapp Swashbuckle.AspNetCore.Swagger latest "$STUBS_PATH"
-          rm -rf ql/test/resources/stubs/_frameworks
-          # Update existing stubs in the repo with the freshly generated ones
-          mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/
-          git status
-          codeql test run --threads=0 --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
+          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Util.Tests"
+          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Extraction.Tests"
+          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests"
+          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"

.github/workflows/csv-coverage-metrics.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
       - name: Create empty database
@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
       - name: Create empty database

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -10,7 +10,6 @@ on:
       - "*/ql/src/**/*.qll"
       - "*/ql/lib/**/*.ql"
       - "*/ql/lib/**/*.qll"
-      - "*/ql/lib/ext/**/*.yml"
       - "misc/scripts/library-coverage/*.py"
       # input data files
       - "*/documentation/library-coverage/cwe-sink.csv"
@@ -31,11 +30,11 @@ jobs:
           GITHUB_CONTEXT: ${{ toJSON(github.event) }}
         run: echo "$GITHUB_CONTEXT"
       - name: Clone self (github/codeql) - MERGE
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: merge
       - name: Clone self (github/codeql) - BASE
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           fetch-depth: 2
           path: base

.github/workflows/csv-coverage-pr-comment.yml

@@ -20,7 +20,7 @@ jobs:
         GITHUB_CONTEXT: ${{ toJSON(github.event) }}
       run: echo "$GITHUB_CONTEXT"
     - name: Clone self (github/codeql)
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
     - name: Set up Python 3.8
       uses: actions/setup-python@v4
       with:

.github/workflows/csv-coverage-timeseries.yml

@@ -9,11 +9,11 @@ jobs:
 
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: script
       - name: Clone self (github/codeql) for analysis
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: codeqlModels
           fetch-depth: 0

.github/workflows/csv-coverage-update.yml

@@ -17,7 +17,7 @@ jobs:
           GITHUB_CONTEXT: ${{ toJSON(github.event) }}
         run: echo "$GITHUB_CONTEXT"
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: ql
           fetch-depth: 0

.github/workflows/csv-coverage.yml

@@ -13,11 +13,11 @@ jobs:
 
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: script
       - name: Clone self (github/codeql) for analysis
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: codeqlModels
           ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}

.github/workflows/fast-forward.yml

@@ -1,50 +0,0 @@
-# Fast-forwards the branch specified in BRANCH_NAME
-# to the github.ref/sha that this workflow is run on.
-# Used as part of the release process, to ensure
-# external query writers can always access a branch of github/codeql
-# that is compatible with the latest stable release.
-name: Fast-forward tracking branch for selected CodeQL version
-on:
-  workflow_dispatch:
-
-jobs:
-  fast-forward:
-    name: Fast-forward tracking branch for selected CodeQL version
-    runs-on: ubuntu-latest
-    if: github.repository == 'github/codeql'
-    permissions:
-      contents: write
-    env:
-      BRANCH_NAME: 'lgtm.com'
-    steps:
-      - name: Validate chosen branch
-        if: ${{ !startsWith(github.ref_name, 'codeql-cli-') }}
-        shell: bash
-        run: |
-          echo "::error ::The $BRANCH_NAME tracking branch should only be fast-forwarded to the tip of a codeql-cli-* branch, got $GITHUB_REF_NAME instead."
-          exit 1
-
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Git config
-        shell: bash
-        run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-
-      - name: Fetch
-        shell: bash
-        run: |
-          set -x
-          echo "Fetching $BRANCH_NAME"
-          # Explicitly unshallow and fetch to ensure the remote ref is available.
-          git fetch --unshallow origin "$BRANCH_NAME"
-          git checkout -b "$BRANCH_NAME" "origin/$BRANCH_NAME"
-
-      - name: Fast-forward
-        shell: bash
-        run: |
-          echo "Fast-forwarding $BRANCH_NAME to ${GITHUB_REF}@${GITHUB_SHA}"
-          git merge --ff-only "$GITHUB_SHA"
-          git push origin "$BRANCH_NAME"

.github/workflows/go-tests-other-os.yml

@@ -7,21 +7,19 @@ on:
       - .github/workflows/go-tests-other-os.yml
       - .github/actions/**
       - codeql-workspace.yml
-env:
-  GO_VERSION: '~1.21.0'
 jobs:
   test-mac:
     name: Test MacOS
     runs-on: macos-latest
     steps:
-      - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v4
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: 1.19
         id: go
 
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
       - name: Set up CodeQL CLI
         uses: ./.github/actions/fetch-codeql
@@ -49,14 +47,14 @@ jobs:
     name: Test Windows
     runs-on: windows-latest-xl
     steps:
-      - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v4
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: 1.19
         id: go
 
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
       - name: Set up CodeQL CLI
         uses: ./.github/actions/fetch-codeql

.github/workflows/go-tests.yml

@@ -15,21 +15,19 @@ on:
       - .github/workflows/go-tests.yml
       - .github/actions/**
       - codeql-workspace.yml
-env:
-  GO_VERSION: '~1.21.0'
 jobs:
   test-linux:
     name: Test Linux (Ubuntu)
     runs-on: ubuntu-latest-xl
     steps:
-      - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v4
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: 1.19
         id: go
 
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
       - name: Set up CodeQL CLI
         uses: ./.github/actions/fetch-codeql

.github/workflows/js-ml-tests.yml

@@ -0,0 +1,65 @@
+name: JS ML-powered queries tests
+
+on:
+  push:
+    paths:
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+      - .github/workflows/js-ml-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+      - "rc/*"
+  pull_request:
+    paths:
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+      - .github/workflows/js-ml-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+  workflow_dispatch:
+
+defaults:
+  run:
+    working-directory: javascript/ql/experimental/adaptivethreatmodeling
+
+jobs:
+  qltest:
+    name: Test QL
+    runs-on: ubuntu-latest-xl
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Install pack dependencies
+        run: |
+          for pack in modelbuilding src test; do
+            codeql pack install --mode verify -- "${pack}"
+          done
+      
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: js-ml-test
+
+      - name: Check QL compilation
+        run: |
+          codeql query compile \
+            --check-only \
+            --ram 50000 \
+            --additional-packs "${{ github.workspace }}" \
+            --threads=0 \
+            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
+            -- \
+            lib modelbuilding src
+
+      - name: Run QL tests
+        run: |
+          codeql test run \
+            --threads=0 \
+            --ram 50000 \
+            --additional-packs "${{ github.workspace }}" \
+            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
+            -- \
+            test

.github/workflows/mad_modelDiff.yml

@@ -27,12 +27,12 @@ jobs:
         slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
     steps:
       - name: Clone github/codeql from PR
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         if: github.event.pull_request
         with:
           path: codeql-pr
       - name: Clone github/codeql from main
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: codeql-main
           ref: main

.github/workflows/mad_regenerate-models.yml

@@ -27,11 +27,11 @@ jobs:
             ref: "placeholder"
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
       - name: Setup CodeQL binaries
         uses: ./.github/actions/fetch-codeql
       - name: Clone repositories
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: repos/${{ matrix.ref }}
           ref: ${{ matrix.ref }}

.github/workflows/qhelp-pr-preview.yml

@@ -43,7 +43,7 @@ jobs:
           if-no-files-found: error
           retention-days: 1
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 2
           persist-credentials: false

.github/workflows/ql-for-ql-build.yml

@@ -5,6 +5,13 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+    paths:
+      - "ql/**"
+      - "**.qll"
+      - "**.ql"
+      - "**.dbscheme"
+      - "**/qlpack.yml"
+      - ".github/workflows/ql-for-ql-build.yml"
 
 env:
   CARGO_TERM_COLOR: always
@@ -14,27 +21,67 @@ jobs:
     runs-on: ubuntu-latest-xl
     steps:
       ### Build the queries ###
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
+      - uses: actions/checkout@v3
+      - name: Find latest bundle
+        id: find-latest-bundle
+        uses: ./.github/actions/find-latest-bundle
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
         with:
           languages: javascript # does not matter
+          tools: ${{ steps.find-latest-bundle.outputs.url }}
+      - name: Get CodeQL version
+        id: get-codeql-version
+        run: |
+          echo "version=$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)" >> $GITHUB_OUTPUT
+        shell: bash
+        env:
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - uses: ./.github/actions/os-version
         id: os_version
+      - name: Cache entire pack
+        id: cache-pack
+        uses: actions/cache@v3
+        with:
+          path: ${{ runner.temp }}/pack
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-pack-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
+      - name: Cache queries
+        if: steps.cache-pack.outputs.cache-hit != 'true'
+        id: cache-queries
+        uses: actions/cache@v3
+        with:
+          path: ${{ runner.temp }}/queries
+          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
+      - name: Build query pack
+        if: steps.cache-queries.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: |
+          cd ql/ql/src
+          "${CODEQL}" pack create -j 16
+          mv .codeql/pack/codeql/ql/0.0.0 ${{ runner.temp }}/queries
+        env:
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+      - name: Move cache queries to pack
+        if: steps.cache-pack.outputs.cache-hit != 'true'
+        run: |
+          cp -r ${{ runner.temp }}/queries ${{ runner.temp }}/pack
+        env:
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+
       ### Build the extractor ###
       - name: Cache entire extractor
+        if: steps.cache-pack.outputs.cache-hit != 'true'
         id: cache-extractor
         uses: actions/cache@v3
         with:
           path: |
-            ql/extractor-pack/
-            ql/target/release/buramu
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ql/**/*.rs') }}
+            ql/target/release/ql-autobuilder
+            ql/target/release/ql-autobuilder.exe
+            ql/target/release/ql-extractor
+            ql/target/release/ql-extractor.exe
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
       - name: Cache cargo
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         uses: actions/cache@v3
         with:
           path: |
@@ -42,33 +89,77 @@ jobs:
             ~/.cargo/git
             ql/target
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
+      - name: Check formatting
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: cd ql; cargo fmt --all -- --check
+      - name: Build
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: cd ql; cargo build --verbose
+      - name: Run tests
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: cd ql; cargo test --verbose
       - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; ./scripts/create-extractor-pack.sh       
-        env:
-          GH_TOKEN: ${{ github.token }}   
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: run-ql-for-ql
-      - name: Make database and analyze
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: cd ql; cargo build --release
+      - name: Generate dbscheme
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
+
+      ### Package the queries and extractor ###
+      - name: Package pack
+        if: steps.cache-pack.outputs.cache-hit != 'true'
         run: |
-          ./ql/target/release/buramu | tee deprecated.blame # Add a blame file for the extractor to parse.
-          ${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB}
-          ${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls  --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-        env: 
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-          DB: ${{ runner.temp }}/DB
-          LGTM_INDEX_FILTERS: |
-            exclude:ql/ql/test
-            exclude:*/ql/lib/upgrades/
-            exclude:java/ql/integration-tests
-      - name: Upload sarif to code-scanning
-        uses: github/codeql-action/upload-sarif@v2
+          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats ${PACK}/
+          mkdir -p ${PACK}/tools/linux64
+          cp ql/target/release/ql-autobuilder  ${PACK}/tools/linux64/autobuilder
+          cp ql/target/release/ql-extractor ${PACK}/tools/linux64/extractor
+          chmod +x ${PACK}/tools/linux64/autobuilder
+          chmod +x ${PACK}/tools/linux64/extractor
+        env:
+          PACK: ${{ runner.temp }}/pack
+
+      ### Run the analysis ###
+      - name: Hack codeql-action options
+        run: |
+          JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .resolve.languages=["--search-path", $pack] | .database.init=["--search-path", $pack]')
+          echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
+        env:
+          PACK: ${{ runner.temp }}/pack
+
+      - name: Create CodeQL config file
+        run: |
+          echo "paths-ignore:" >> ${CONF}
+          echo "  - ql/ql/test" >> ${CONF}
+          echo "  - \"*/ql/lib/upgrades/\"" >> ${CONF}
+          echo "disable-default-queries: true" >> ${CONF}
+          echo "queries:" >> ${CONF}
+          echo "  - uses: ./ql/ql/src/codeql-suites/ql-code-scanning.qls" >> ${CONF}
+          echo "Config file: "
+          cat ${CONF}
+        env:
+          CONF: ./ql-for-ql-config.yml
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
         with:
-          sarif_file: ql-for-ql.sarif
-          category: ql-for-ql
+          languages: ql
+          db-location: ${{ runner.temp }}/db
+          config-file: ./ql-for-ql-config.yml
+          tools: ${{ steps.find-latest-bundle.outputs.url }}
+      - name: Move pack queries
+        run: |
+          cp -r ${PACK}/queries ql/ql/src
+        env:
+          PACK: ${{ runner.temp }}/pack
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@45955cb1830b640e2c1603ad72ad542a49d47b96
+        with:
+          category: "ql-for-ql"
+      - name: Copy sarif file to CWD
+        run: cp ../results/ql.sarif ./ql-for-ql.sarif
+      - name: Fixup the $scema in sarif  # Until https://github.com/microsoft/sarif-vscode-extension/pull/436/ is part in a stable release
+        run: |
+          sed -i 's/\$schema.*/\$schema": "https:\/\/raw.githubusercontent.com\/oasis-tcs\/sarif-spec\/master\/Schemata\/sarif-schema-2.1.0",/' ql-for-ql.sarif
       - name: Sarif as artifact
         uses: actions/upload-artifact@v3
         with:

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -21,11 +21,11 @@ jobs:
           - github/codeql
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version
@@ -42,7 +42,7 @@ jobs:
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           repository: ${{ matrix.repo }}
           path: ${{ github.workspace }}/repo
@@ -71,7 +71,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: measure
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
         with:
           name: measurements

.github/workflows/ql-for-ql-tests.yml

@@ -6,13 +6,11 @@ on:
     paths:
       - "ql/**"
       - codeql-workspace.yml
-      - .github/workflows/ql-for-ql-tests.yml
   pull_request:
     branches: [main]
     paths:
       - "ql/**"
       - codeql-workspace.yml
-      - .github/workflows/ql-for-ql-tests.yml
 
 env:
   CARGO_TERM_COLOR: always
@@ -21,10 +19,10 @@ jobs:
   qltest:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version
@@ -36,8 +34,6 @@ jobs:
             ~/.cargo/git
             ql/target
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/rust-toolchain.toml', 'ql/**/Cargo.lock') }}
-      - name: Check formatting
-        run: cd ql; cargo fmt --all -- --check
       - name: Build extractor
         run: |
           cd ql;
@@ -61,7 +57,7 @@ jobs:
     needs: [qltest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Install GNU tar 
         if: runner.os == 'macOS'
         run: |
@@ -69,7 +65,7 @@ jobs:
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version

.github/workflows/query-list.yml

@@ -20,7 +20,7 @@ jobs:
 
     steps:
     - name: Clone self (github/codeql)
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
       with:
         path: codeql 
     - name: Set up Python 3.8

.github/workflows/ruby-build.yml

@@ -42,15 +42,12 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Install GNU tar
         if: runner.os == 'macOS'
         run: |
           brew install gnu-tar
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
-      - name: Install cargo-cross
-        if: runner.os == 'Linux'
-        run: cargo install cross --version 0.2.5
       - uses: ./.github/actions/os-version
         id: os_version
       - name: Cache entire extractor
@@ -58,10 +55,12 @@ jobs:
         id: cache-extractor
         with:
           path: |
-            ruby/extractor/target/release/codeql-extractor-ruby
-            ruby/extractor/target/release/codeql-extractor-ruby.exe
-            ruby/extractor/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ruby/extractor/**/*.rs') }}
+            ruby/target/release/ruby-autobuilder
+            ruby/target/release/ruby-autobuilder.exe
+            ruby/target/release/ruby-extractor
+            ruby/target/release/ruby-extractor.exe
+            ruby/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}--${{ hashFiles('ruby/**/*.rs') }}
       - uses: actions/cache@v3
         if: steps.cache-extractor.outputs.cache-hit != 'true'
         with:
@@ -69,30 +68,22 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             ruby/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-rust-cargo-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/**/Cargo.lock') }}
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-rust-cargo-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}
       - name: Check formatting
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd extractor && cargo fmt --all -- --check
+        run: cargo fmt --all -- --check
       - name: Build
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd extractor && cargo build --verbose
+        run: cargo build --verbose
       - name: Run tests
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd extractor && cargo test --verbose
-      # On linux, build the extractor via cross in a centos7 container.
-      # This ensures we don't depend on glibc > 2.17.
-      - name: Release build (linux)
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os == 'Linux'
-        run: |
-          cd extractor
-          cross build --release
-          mv target/x86_64-unknown-linux-gnu/release/codeql-extractor-ruby target/release/
-      - name: Release build (windows and macos)
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os != 'Linux'
-        run: cd extractor && cargo build --release
+        run: cargo test --verbose
+      - name: Release build
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        run: cargo build --release
       - name: Generate dbscheme
         if: ${{ matrix.os == 'ubuntu-latest' && steps.cache-extractor.outputs.cache-hit != 'true'}}
-        run: extractor/target/release/codeql-extractor-ruby generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+        run: target/release/ruby-generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v3
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
@@ -107,13 +98,15 @@ jobs:
         with:
           name: extractor-${{ matrix.os }}
           path: |
-            ruby/extractor/target/release/codeql-extractor-ruby
-            ruby/extractor/target/release/codeql-extractor-ruby.exe
+            ruby/target/release/ruby-autobuilder
+            ruby/target/release/ruby-autobuilder.exe
+            ruby/target/release/ruby-extractor
+            ruby/target/release/ruby-extractor.exe
           retention-days: 1
   compile-queries:
     runs-on: ubuntu-latest-xl
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Fetch CodeQL
         uses: ./.github/actions/fetch-codeql
       - name: Cache compilation cache
@@ -123,29 +116,28 @@ jobs:
           key: ruby-build
       - name: Build Query Pack
         run: |
-          PACKS=${{ runner.temp }}/query-packs
-          rm -rf $PACKS
-          codeql pack create ../misc/suite-helpers --output "$PACKS"
-          codeql pack create ../shared/regex --output "$PACKS"
-          codeql pack create ../shared/ssa --output "$PACKS"
-          codeql pack create ../shared/tutorial --output "$PACKS"
-          codeql pack create ql/lib --output "$PACKS"
-          codeql pack create -j0 ql/src --output "$PACKS" --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-          PACK_FOLDER=$(readlink -f "$PACKS"/codeql/ruby-queries/*)
+          rm -rf target/packs
+          codeql pack create ../misc/suite-helpers --output target/packs
+          codeql pack create ../shared/regex --output target/packs
+          codeql pack create ../shared/ssa --output target/packs
+          codeql pack create ../shared/tutorial --output target/packs
+          codeql pack create ql/lib --output target/packs
+          codeql pack create -j0 ql/src --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
           codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
           (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
       - uses: actions/upload-artifact@v3
         with:
           name: codeql-ruby-queries
           path: |
-            ${{ runner.temp }}/query-packs/*
+            ruby/target/packs/*
           retention-days: 1
 
   package:
     runs-on: ubuntu-latest
     needs: [build, compile-queries]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
         with:
           name: ruby.dbscheme
@@ -166,10 +158,13 @@ jobs:
           mkdir -p ruby
           cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
           mkdir -p ruby/tools/{linux64,osx64,win64}
-          cp linux64/codeql-extractor-ruby ruby/tools/linux64/extractor
-          cp osx64/codeql-extractor-ruby ruby/tools/osx64/extractor
-          cp win64/codeql-extractor-ruby.exe ruby/tools/win64/extractor.exe
-          chmod +x ruby/tools/{linux64,osx64}/extractor
+          cp linux64/ruby-autobuilder ruby/tools/linux64/autobuilder
+          cp osx64/ruby-autobuilder ruby/tools/osx64/autobuilder
+          cp win64/ruby-autobuilder.exe ruby/tools/win64/autobuilder.exe
+          cp linux64/ruby-extractor ruby/tools/linux64/extractor
+          cp osx64/ruby-extractor ruby/tools/osx64/extractor
+          cp win64/ruby-extractor.exe ruby/tools/win64/extractor.exe
+          chmod +x ruby/tools/{linux64,osx64}/{autobuilder,extractor}
           zip -rq codeql-ruby.zip ruby
       - uses: actions/upload-artifact@v3
         with:
@@ -206,7 +201,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     needs: [package]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Fetch CodeQL
         uses: ./.github/actions/fetch-codeql
 
@@ -231,54 +226,3 @@ jobs:
         shell: bash
         run: |
           codeql database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls
-
-  # This is a copy of the 'test' job that runs in a centos7 container.
-  # This tests that the extractor works correctly on systems with an old glibc.
-  test-centos7:
-    defaults:
-      run:
-        working-directory: ${{ github.workspace }}
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-latest
-    container:
-      image: centos:centos7
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    needs: [package]
-    steps:
-      - name: Install gh cli
-        run: |
-          yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
-          # fetch-codeql requires unzip and jq
-          # jq is available in epel-release (https://docs.fedoraproject.org/en-US/epel/)
-          yum install -y gh unzip epel-release
-          yum install -y jq
-      - uses: actions/checkout@v3
-      - name: Fetch CodeQL
-        uses: ./.github/actions/fetch-codeql
-
-      # Due to a bug in Actions, we can't use runner.temp in the run blocks here.
-      # https://github.com/actions/runner/issues/2185
-
-      - name: Download Ruby bundle
-        uses: actions/download-artifact@v3
-        with:
-          name: codeql-ruby-bundle
-          path: ${{ runner.temp }}
-      - name: Unzip Ruby bundle
-        shell: bash
-        run: unzip -q -d "$RUNNER_TEMP"/ruby-bundle "$RUNNER_TEMP"/codeql-ruby-bundle.zip
-
-      - name: Run QL test
-        shell: bash
-        run: |
-          codeql test run --search-path "$RUNNER_TEMP"/ruby-bundle --additional-packs "$RUNNER_TEMP"/ruby-bundle ruby/ql/test/library-tests/ast/constants/
-      - name: Create database
-        shell: bash
-        run: |
-          codeql database create --search-path "$RUNNER_TEMP"/ruby-bundle --language ruby --source-root ruby/ql/test/library-tests/ast/constants/ ../database
-      - name: Analyze database
-        shell: bash
-        run: |
-          codeql database analyze --search-path "$RUNNER_TEMP"/ruby-bundle --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/ruby-dataset-measure.yml

@@ -27,14 +27,14 @@ jobs:
         repo: [rails/rails, discourse/discourse, spree/spree, ruby/ruby]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
 
       - uses: ./.github/actions/fetch-codeql
 
       - uses: ./ruby/actions/create-extractor-pack
 
       - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           repository: ${{ matrix.repo }}
           path: ${{ github.workspace }}/repo
@@ -59,7 +59,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: measure
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
         with:
           name: measurements

.github/workflows/ruby-qltest.yml

@@ -4,7 +4,6 @@ on:
   push:
     paths:
       - "ruby/**"
-      - "shared/**"
       - .github/workflows/ruby-build.yml
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
@@ -14,7 +13,6 @@ on:
   pull_request:
     paths:
       - "ruby/**"
-      - "shared/**"
       - .github/workflows/ruby-qltest.yml
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
@@ -33,7 +31,7 @@ jobs:
   qlupgrade:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - name: Check DB upgrade scripts
         run: |
@@ -54,7 +52,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - uses: ./ruby/actions/create-extractor-pack
       - name: Cache compilation cache

.github/workflows/swift.yml

@@ -5,7 +5,6 @@ on:
     paths:
       - "swift/**"
       - "misc/bazel/**"
-      - "misc/codegen/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**
@@ -16,12 +15,10 @@ on:
     branches:
       - main
       - rc/*
-      - codeql-cli-*
   push:
     paths:
       - "swift/**"
       - "misc/bazel/**"
-      - "misc/codegen/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**
@@ -31,7 +28,6 @@ on:
     branches:
       - main
       - rc/*
-      - codeql-cli-*
 
 jobs:
   # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks
@@ -39,31 +35,31 @@ jobs:
   build-and-test-macos:
     runs-on: macos-12-xl
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./swift/actions/build-and-test
   build-and-test-linux:
     runs-on: ubuntu-latest-xl
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./swift/actions/build-and-test
   qltests-linux:
     needs: build-and-test-linux
     runs-on: ubuntu-latest-xl
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./swift/actions/run-ql-tests
   qltests-macos:
     if : ${{ github.event_name == 'pull_request' }}
     needs: build-and-test-macos
     runs-on: macos-12-xl
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./swift/actions/run-ql-tests
   integration-tests-linux:
     needs: build-and-test-linux
     runs-on: ubuntu-latest-xl
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./swift/actions/run-integration-tests
   integration-tests-macos:
     if : ${{ github.event_name == 'pull_request' }}
@@ -71,13 +67,13 @@ jobs:
     runs-on: macos-12-xl
     timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./swift/actions/run-integration-tests
   codegen:
     if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: bazelbuild/setup-bazelisk@v2
       - uses: actions/setup-python@v4
         with:
@@ -102,6 +98,6 @@ jobs:
     if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - uses: ./swift/actions/database-upgrade-scripts

.github/workflows/sync-files.yml

@@ -14,9 +14,7 @@ jobs:
   sync:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Check synchronized files
         run: python config/sync-files.py
-      - name: Check dbscheme fragments
-        run: python config/sync-dbscheme-fragments.py
 

.github/workflows/tree-sitter-extractor-test.yml

@@ -1,46 +0,0 @@
-name: Test tree-sitter-extractor
-
-on:
-  push:
-    paths:
-      - "shared/tree-sitter-extractor/**"
-      - .github/workflows/tree-sitter-extractor-test.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "shared/tree-sitter-extractor/**"
-      - .github/workflows/tree-sitter-extractor-test.yml
-    branches:
-      - main
-      - "rc/*"
-
-env:
-  CARGO_TERM_COLOR: always
-
-defaults:
-  run:
-    working-directory: shared/tree-sitter-extractor
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Check formatting
-        run: cargo fmt --all -- --check
-      - name: Run tests
-        run: cargo test --verbose
-  fmt:
-    runs-on: ubuntu-latest  
-    steps:
-      - uses: actions/checkout@v4
-      - name: Check formatting
-        run: cargo fmt --check
-  clippy:
-    runs-on: ubuntu-latest  
-    steps:
-      - uses: actions/checkout@v4
-      - name: Run clippy
-        run: cargo clippy -- --no-deps -D warnings -A clippy::new_without_default -A clippy::too_many_arguments

.github/workflows/validate-change-notes.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql

.pre-commit-config.yaml

@@ -5,9 +5,9 @@ repos:
     rev: v3.2.0
     hooks:
       - id: trailing-whitespace
-        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)|.*\.patch
+        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
       - id: end-of-file-fixer
-        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)|.*\.patch
+        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: v13.0.1
@@ -19,12 +19,7 @@ repos:
     rev: v1.6.0
     hooks:
       - id: autopep8
-        files: ^misc/codegen/.*\.py
-
-  - repo: https://github.com/warchant/pre-commit-buildifier
-    rev: 0.0.2
-    hooks:
-      - id: buildifier
+        files: ^swift/.*\.py
 
   - repo: local
     hooks:
@@ -58,5 +53,5 @@ repos:
         name: Run Swift code generation unit tests
         files: ^swift/codegen/.*\.py$
         language: system
-        entry: bazel test //misc/codegen/test
+        entry: bazel test //swift/codegen/test
         pass_filenames: false

.vscode/tasks.json

@@ -22,22 +22,6 @@
                 "command": "${config:python.pythonPath}",
             },
             "problemMatcher": []
-        },
-        {
-            "label": "Accept .expected changes from CI",
-            "type": "process",
-            // Non-Windows OS will usually have Python 3 already installed at /usr/bin/python3.
-            "command": "python3",
-            "args": [
-                "misc/scripts/accept-expected-changes-from-ci.py"
-            ],
-            "group": "build",
-            "windows": {
-                // On Windows, use whatever Python interpreter is configured for this workspace. The default is
-                // just `python`, so if Python is already on the path, this will find it.
-                "command": "${config:python.pythonPath}",
-            },
-            "problemMatcher": []
         }
     ]
-}
+}

CODEOWNERS

@@ -6,8 +6,8 @@
 /python/ @github/codeql-python
 /ruby/ @github/codeql-ruby
 /swift/ @github/codeql-swift
-/misc/codegen/ @github/codeql-swift
 /java/kotlin-extractor/ @github/codeql-kotlin
+/java/kotlin-explorer/ @github/codeql-kotlin
 
 # ML-powered queries
 /javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
@@ -39,6 +39,3 @@ WORKSPACE.bazel @github/codeql-ci-reviewers
 /.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
 /.github/workflows/ruby-* @github/codeql-ruby
 /.github/workflows/swift.yml @github/codeql-swift
-
-# Misc
-/misc/scripts/accept-expected-changes-from-ci.py @RasmusWL

CONTRIBUTING.md

@@ -14,16 +14,14 @@ If you have an idea for a query that you would like to share with other CodeQL u
 
 1. **Directory structure**
 
-    There are eight language-specific query directories in this repository:
+    There are six language-specific query directories in this repository:
 
       * C/C++: `cpp/ql/src`
       * C#: `csharp/ql/src`
-      * Go: `go/ql/src`
-      * Java/Kotlin: `java/ql/src`
+      * Java: `java/ql/src`
       * JavaScript: `javascript/ql/src`
       * Python: `python/ql/src`
       * Ruby: `ruby/ql/src`
-      * Swift: `swift/ql/src`
 
     Each language-specific directory contains further subdirectories that group queries based on their `@tags` or purpose.
     - Experimental queries and libraries are stored in the `experimental` subdirectory within each language-specific directory in the [CodeQL repository](https://github.com/github/codeql). For example, experimental Java queries and libraries are stored in `java/ql/src/experimental` and any corresponding tests in `java/ql/test/experimental`.

codeql-workspace.yml

@@ -4,22 +4,20 @@ provide:
   - "*/ql/test/qlpack.yml"
   - "*/ql/examples/qlpack.yml"
   - "*/ql/consistency-queries/qlpack.yml"
-  - "*/ql/automodel/src/qlpack.yml"
-  - "*/ql/automodel/test/qlpack.yml"
   - "shared/*/qlpack.yml"
   - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
   - "go/ql/config/legacy-support/qlpack.yml"
   - "go/build/codeql-extractor-go/codeql-extractor.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml"
+  - "*/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml"
   # This pack is explicitly excluded from the workspace since most users
   # will want to use a version of this pack from the package cache. Internal
   # users can uncomment the following line and place a custom ML model
   # in the corresponding pack to test a custom ML model within their local
   # checkout.
-  # - "javascript/ql/experimental/adaptivethreatmodeling/model/qlpack.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/test/qlpack.yml"
+  - "*/ql/experimental/adaptivethreatmodeling/model/qlpack.yml"
+  - "*/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml"
+  - "*/ql/experimental/adaptivethreatmodeling/src/qlpack.yml"
+  - "*/ql/experimental/adaptivethreatmodeling/test/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/lib/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/src/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/test/qlpack.yml"

config/dbscheme-fragments.json

@@ -1,33 +0,0 @@
-{
-  "files": [
-    "javascript/ql/lib/semmlecode.javascript.dbscheme",
-    "python/ql/lib/semmlecode.python.dbscheme",
-    "ruby/ql/lib/ruby.dbscheme",
-    "ql/ql/src/ql.dbscheme"
-  ],
-  "fragments": [
-    "/*- External data -*/",
-    "/*- Files and folders -*/",
-    "/*- Diagnostic messages -*/",
-    "/*- Diagnostic messages: severity -*/",
-    "/*- Source location prefix -*/",
-    "/*- Lines of code -*/",
-    "/*- Configuration files with key value pairs -*/",
-    "/*- YAML -*/",
-    "/*- XML Files -*/",
-    "/*- XML: sourceline -*/",
-    "/*- DEPRECATED: External defects and metrics -*/",
-    "/*- DEPRECATED: Snapshot date -*/",
-    "/*- DEPRECATED: Duplicate code -*/",
-    "/*- DEPRECATED: Version control data -*/",
-    "/*- JavaScript-specific part -*/",
-    "/*- Ruby dbscheme -*/",
-    "/*- Erb dbscheme -*/",
-    "/*- QL dbscheme -*/",
-    "/*- Dbscheme dbscheme -*/",
-    "/*- Yaml dbscheme -*/",
-    "/*- Blame dbscheme -*/",
-    "/*- JSON dbscheme -*/",
-    "/*- Python dbscheme -*/"
-  ]
-}

config/identical-files.json

@@ -1,41 +1,66 @@
 {
-  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift Legacy Configuration": [
-    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl1.qll",
+  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift": [
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl2.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl3.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl4.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl5.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl6.qll",
-    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl1.qll",
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForSerializability.qll",
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForOnActivityResult.qll",
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl4.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplLocal.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl1.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl1.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl2.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl1.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplForContentDataFlow.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll",
     "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl2.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl1.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImplForStringsNewReplacer.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl1.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl1.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForRegExp.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForHttpClientLibraries.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForPathname.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"
   ],
-  "TaintTracking Legacy Configuration Java/C++/C#/Go/Python/Ruby/Swift": [
+  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift Common": [
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll",
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImplCommon.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll"
+  ],
+  "TaintTracking::Configuration Java/C++/C#/Go/Python/Ruby/Swift": [
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
@@ -53,6 +78,16 @@
     "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
   ],
+  "DataFlow Java/C++/C#/Python/Ruby/Swift Consistency checks": [
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll",
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplConsistency.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplConsistency.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplConsistency.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplConsistency.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplConsistency.qll"
+  ],
   "DataFlow Java/C#/Go/Ruby/Python/Swift Flow Summaries": [
     "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll",
@@ -69,10 +104,6 @@
     "java/ql/src/utils/modelgenerator/internal/CaptureModels.qll",
     "csharp/ql/src/utils/modelgenerator/internal/CaptureModels.qll"
   ],
-  "Model as Data Generation Java/C# - CaptureModelsPrinting": [
-    "java/ql/src/utils/modelgenerator/internal/CaptureModelsPrinting.qll",
-    "csharp/ql/src/utils/modelgenerator/internal/CaptureModelsPrinting.qll"
-  ],
   "Sign Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/Sign.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/Sign.qll"
@@ -229,11 +260,6 @@
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/IRBlockImports.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/IRBlockImports.qll"
   ],
-  "C++ IR IRConsistencyImports": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/IRConsistencyImports.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/IRConsistencyImports.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/IRConsistencyImports.qll"
-  ],
   "C++ IR IRFunctionImports": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/IRFunctionImports.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/IRFunctionImports.qll",
@@ -459,17 +485,17 @@
   "SensitiveDataHeuristics Python/JS": [
     "javascript/ql/lib/semmle/javascript/security/internal/SensitiveDataHeuristics.qll",
     "python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll",
-    "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll",
-    "swift/ql/lib/codeql/swift/security/internal/SensitiveDataHeuristics.qll"
+    "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll"
+  ],
+  "CFG": [
+    "csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll",
+    "ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll",
+    "swift/ql/lib/codeql/swift/controlflow/internal/ControlFlowGraphImplShared.qll"
   ],
   "TypeTracker": [
     "python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll",
     "ruby/ql/lib/codeql/ruby/typetracking/TypeTracker.qll"
   ],
-  "SummaryTypeTracker": [
-    "python/ql/lib/semmle/python/dataflow/new/internal/SummaryTypeTracker.qll",
-    "ruby/ql/lib/codeql/ruby/typetracking/internal/SummaryTypeTracker.qll"
-  ],
   "AccessPathSyntax": [
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/AccessPathSyntax.qll",
     "go/ql/lib/semmle/go/dataflow/internal/AccessPathSyntax.qll",
@@ -545,9 +571,5 @@
   "EncryptionKeySizes Python/Java": [
     "python/ql/lib/semmle/python/security/internal/EncryptionKeySizes.qll",
     "java/ql/lib/semmle/code/java/security/internal/EncryptionKeySizes.qll"
-  ],
-  "Python model summaries test extension": [
-    "python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ext.yml",
-    "python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ext.yml"
   ]
 }

config/sync-dbscheme-fragments.py

@@ -1,86 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import json
-import os
-import pathlib
-import re
-
-
-def make_groups(blocks):
-    groups = {}
-    for block in blocks:
-        groups.setdefault("".join(block["lines"]), []).append(block)
-    return list(groups.values())
-
-
-def validate_fragments(fragments):
-    ok = True
-    for header, blocks in fragments.items():
-        groups = make_groups(blocks)
-        if len(groups) > 1:
-            ok = False
-            print("Warning: dbscheme fragments with header '{}' are different for {}".format(header, ["{}:{}:{}".format(
-                group[0]["file"], group[0]["start"], group[0]["end"]) for group in groups]))
-    return ok
-
-
-def main():
-    script_path = os.path.realpath(__file__)
-    script_dir = os.path.dirname(script_path)
-    parser = argparse.ArgumentParser(
-        prog=os.path.basename(script_path),
-        description='Sync dbscheme fragments across files.'
-    )
-    parser.add_argument('files', metavar='dbscheme_file', type=pathlib.Path, nargs='*', default=[],
-                        help='dbscheme files to check')
-    args = parser.parse_args()
-
-    with open(os.path.join(script_dir, "dbscheme-fragments.json"), "r") as f:
-        config = json.load(f)
-
-    fragment_headers = set(config["fragments"])
-    fragments = {}
-    ok = True
-    for file in args.files + config["files"]:
-        with open(os.path.join(os.path.dirname(script_dir), file), "r") as dbscheme:
-            header = None
-            line_number = 1
-            block = {"file": file, "start": line_number,
-                     "end": None, "lines": []}
-
-            def end_block():
-                block["end"] = line_number - 1
-                if len(block["lines"]) > 0:
-                    if header is None:
-                        if re.match(r'(?m)\A(\s|//.*$|/\*(\**[^\*])*\*+/)*\Z', "".join(block["lines"])):
-                            # Ignore comments at the beginning of the file
-                            pass
-                        else:
-                            ok = False
-                            print("Warning: dbscheme fragment without header: {}:{}:{}".format(
-                                block["file"], block["start"], block["end"]))
-                    else:
-                        fragments.setdefault(header, []).append(block)
-            for line in dbscheme:
-                m = re.match(r"^\/\*-.*-\*\/$", line)
-                if m:
-                    end_block()
-                    header = line.strip()
-                    if header not in fragment_headers:
-                        ok = False
-                        print("Warning: unknown header for dbscheme fragment: '{}': {}:{}".format(
-                            header, file, line_number))
-                    block = {"file": file, "start": line_number,
-                             "end": None, "lines": []}
-                block["lines"].append(line)
-                line_number += 1
-            block["lines"].append('\n')
-            line_number += 1
-            end_block()
-    if not ok or not validate_fragments(fragments):
-        exit(1)
-
-
-if __name__ == "__main__":
-    main()

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -1,6 +1,5 @@
 using Xunit;
 using Semmle.Autobuild.Shared;
-using Semmle.Util;
 using System.Collections.Generic;
 using System;
 using System.Linq;
@@ -76,15 +75,6 @@ namespace Semmle.Autobuild.Cpp.Tests
             throw new ArgumentException("Missing RunProcess " + pattern);
         }
 
-        int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
-        {
-            var ret = (this as IBuildActions).RunProcess(cmd, args, workingDirectory, env, out var stdout);
-
-            stdout.ForEach(line => onOutput(line));
-
-            return ret;
-        }
-
         public IList<string> DirectoryDeleteIn = new List<string>();
 
         void IBuildActions.DirectoryDelete(string dir, bool recursive)
@@ -141,14 +131,6 @@ namespace Semmle.Autobuild.Cpp.Tests
 
         bool IBuildActions.IsWindows() => IsWindows;
 
-        public bool IsMacOs { get; set; }
-
-        bool IBuildActions.IsMacOs() => IsMacOs;
-
-        public bool IsArm { get; set; }
-
-        bool IBuildActions.IsArm() => IsArm;
-
         string IBuildActions.PathCombine(params string[] parts)
         {
             return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));
@@ -194,15 +176,6 @@ namespace Semmle.Autobuild.Cpp.Tests
             if (!DownloadFiles.Contains((address, fileName)))
                 throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");
         }
-
-        public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new TestDiagnosticWriter();
-    }
-
-    internal class TestDiagnosticWriter : IDiagnosticsWriter
-    {
-        public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
-
-        public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
     }
 
     /// <summary>
@@ -262,7 +235,6 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
-            Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
             Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
@@ -327,7 +299,7 @@ namespace Semmle.Autobuild.Cpp.Tests
         {
             Actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test.sln -DisableParallelProcessing"] = 1;
             Actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test.sln -DisableParallelProcessing"] = 0;
-            Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program^ Files^ ^(x86^)\Microsoft^ Visual^ Studio^ 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && msbuild C:\Project\test.sln /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"""] = 0;
+            Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && msbuild C:\Project\test.sln /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"""] = 0;
             Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0;

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
     <Nullable>enable</Nullable>

cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs

@@ -1,5 +1,4 @@
 using Semmle.Autobuild.Shared;
-using Semmle.Util;
 
 namespace Semmle.Autobuild.Cpp
 {
@@ -22,7 +21,7 @@ namespace Semmle.Autobuild.Cpp
 
     public class CppAutobuilder : Autobuilder<CppAutobuildOptions>
     {
-        public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options, new DiagnosticClassifier()) { }
+        public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options) { }
 
         public override BuildScript GetBuildScript()
         {

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <AssemblyName>Semmle.Autobuild.Cpp</AssemblyName>
     <RootNamespace>Semmle.Autobuild.Cpp</RootNamespace>
     <ApplicationIcon />

cpp/downgrades/19887dbd33327fb07d54251786e0cb2578539775/upgrade.properties

@@ -1,4 +0,0 @@
-description: Revert support for repeated initializers, which are allowed in C with designated initializers.
-compatibility: full
-aggregate_field_init.rel: reorder aggregate_field_init.rel (int aggregate, int initializer, int field, int position) aggregate initializer field
-aggregate_array_init.rel: reorder aggregate_array_init.rel (int aggregate, int initializer, int element_index, int position) aggregate initializer element_index

cpp/downgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/upgrade.properties

@@ -1,2 +0,0 @@
-description: Remove _Float128 type
-compatibility: full

cpp/downgrades/dbe9c8eb5fc6f54b7ae08c7317d0795b24961564/upgrade.properties

@@ -1,2 +0,0 @@
-description: Make __is_trivial a builtin operation
-compatibility: full

cpp/downgrades/qlpack.yml

@@ -2,4 +2,3 @@ name: codeql/cpp-downgrades
 groups: cpp
 downgrades: .
 library: true
-warnOnImplicitThis: true

cpp/ql/examples/qlpack.yml

@@ -4,4 +4,3 @@ groups:
   - examples
 dependencies:
   codeql/cpp-all: ${workspace}
-warnOnImplicitThis: true

cpp/ql/examples/queries.xml

@@ -0,0 +1 @@
+<queries language="cpp"/>

cpp/ql/lib/CHANGELOG.md

@@ -1,188 +1,3 @@
-## 0.9.3
-
-No user-facing changes.
-
-## 0.9.2
-
-### Deprecated APIs
-
-* `getAllocatorCall` on `DeleteExpr` and `DeleteArrayExpr` has been deprecated. `getDeallocatorCall` should be used instead.
-
-### New Features
-
-* Added `DeleteOrDeleteArrayExpr` as a super type of `DeleteExpr` and `DeleteArrayExpr`
-
-### Minor Analysis Improvements
-
-* `delete` and `delete[]` are now modeled as calls to the relevant `operator delete` in the IR. In the case of a dynamic delete call a new instruction `VirtualDeleteFunctionAddress` is used to represent a function that dispatches to the correct delete implementation.
-* Only the 2 level indirection of `argv` (corresponding to `**argv`) is consided for `FlowSource`.
-
-## 0.9.1
-
-No user-facing changes.
-
-## 0.9.0
-
-### Breaking Changes
-
-* The `shouldPrintFunction` predicate from `PrintAstConfiguration` has been replaced by `shouldPrintDeclaration`. Users should now override `shouldPrintDeclaration` if they want to limit the declarations that should be printed.
-* The `shouldPrintFunction` predicate from `PrintIRConfiguration` has been replaced by `shouldPrintDeclaration`. Users should now override `shouldPrintDeclaration` if they want to limit the declarations that should be printed.
-
-### Major Analysis Improvements
-
-* The `PrintAST` library now also prints global and namespace variables and their initializers.
-
-### Minor Analysis Improvements
-
-* The `_Float128x` type is no longer exposed as a builtin type. As this type could not occur any code base, this should only affect queries that explicitly looked at the builtin types.
-
-## 0.8.1
-
-### Deprecated APIs
-
-* The library `semmle.code.cpp.dataflow.DataFlow` has been deprecated. Please use `semmle.code.cpp.dataflow.new.DataFlow` instead.
-
-### New Features
-
-* The `DataFlow::StateConfigSig` signature module has gained default implementations for `isBarrier/2` and `isAdditionalFlowStep/4`. 
-  Hence it is no longer needed to provide `none()` implementations of these predicates if they are not needed.
-
-### Minor Analysis Improvements
-
-* Data flow configurations can now include a predicate `neverSkip(Node node)`
-  in order to ensure inclusion of certain nodes in the path explanations. The
-  predicate defaults to the end-points of the additional flow steps provided in
-  the configuration, which means that such steps now always are visible by
-  default in path explanations.
-* The `IRGuards` library has improved handling of pointer addition and subtraction operations.
-
-## 0.8.0
-
-### New Features
-
-* The `ProductFlow::StateConfigSig` signature now includes default predicates for `isBarrier1`, `isBarrier2`, `isAdditionalFlowStep1`, and `isAdditionalFlowStep1`. Hence, it is no longer needed to provide `none()` implementations of these predicates if they are not needed.
-
-### Minor Analysis Improvements
-
-* Deleted the deprecated `getURL` predicate from the `Container`, `Folder`, and `File` classes. Use the `getLocation` predicate instead.
-
-## 0.7.4
-
-No user-facing changes.
-
-## 0.7.3
-
-### Minor Analysis Improvements
-
-* Deleted the deprecated `hasCopyConstructor` predicate from the `Class` class in `Class.qll`.
-* Deleted many deprecated predicates and classes with uppercase `AST`, `SSA`, `CFG`, `API`, etc. in their names. Use the PascalCased versions instead.
-* Deleted the deprecated `CodeDuplication.qll` file.
-
-## 0.7.2
-
-### New Features
-
-* Added an AST-based interface (`semmle.code.cpp.rangeanalysis.new.RangeAnalysis`) for the relative range analysis library.
-* A new predicate `BarrierGuard::getAnIndirectBarrierNode` has been added to the new dataflow library (`semmle.code.cpp.dataflow.new.DataFlow`) to mark indirect expressions as barrier nodes using the `BarrierGuard` API.
-
-### Major Analysis Improvements
-
-* In the intermediate representation, handling of control flow after non-returning calls has been improved. This should remove false positives in queries that use the intermedite representation or libraries based on it, including the new data flow library.
-
-### Minor Analysis Improvements
-
-* The `StdNamespace` class now also includes all inline namespaces that are children of `std` namespace.
-* The new dataflow (`semmle.code.cpp.dataflow.new.DataFlow`) and taint-tracking libraries (`semmle.code.cpp.dataflow.new.TaintTracking`) now support tracking flow through static local variables.
-
-## 0.7.1
-
-No user-facing changes.
-
-## 0.7.0
-
-### Breaking Changes
-
-* The internal `SsaConsistency` module has been moved from `SSAConstruction` to `SSAConsitency`, and the deprecated `SSAConsistency` module has been removed.
-
-### Deprecated APIs
-
-* The single-parameter predicates `ArrayOrVectorAggregateLiteral.getElementExpr` and `ClassAggregateLiteral.getFieldExpr` have been deprecated in favor of `ArrayOrVectorAggregateLiteral.getAnElementExpr` and `ClassAggregateLiteral.getAFieldExpr`.
-* The recently introduced new data flow and taint tracking APIs have had a
-  number of module and predicate renamings. The old APIs remain in place for
-  now.
-* The `SslContextCallAbstractConfig`, `SslContextCallConfig`, `SslContextCallBannedProtocolConfig`, `SslContextCallTls12ProtocolConfig`, `SslContextCallTls13ProtocolConfig`, `SslContextCallTlsProtocolConfig`, `SslContextFlowsToSetOptionConfig`, `SslOptionConfig` dataflow configurations from `BoostorgAsio` have been deprecated. Please use `SslContextCallConfigSig`, `SslContextCallGlobal`, `SslContextCallFlow`, `SslContextCallBannedProtocolFlow`, `SslContextCallTls12ProtocolFlow`, `SslContextCallTls13ProtocolFlow`, `SslContextCallTlsProtocolFlow`, `SslContextFlowsToSetOptionFlow`.
-
-### New Features
-
-* Added overridable predicates `getSizeExpr` and `getSizeMult` to the `BufferAccess` class (`semmle.code.cpp.security.BufferAccess.qll`). This makes it possible to model a larger class of buffer reads and writes using the library.
-
-### Minor Analysis Improvements
-
-* The `BufferAccess` library (`semmle.code.cpp.security.BufferAccess`) no longer matches buffer accesses inside unevaluated contexts (such as inside `sizeof` or `decltype` expressions). As a result, queries using this library may see fewer false positives.
-
-### Bug Fixes
-
-* Fixed some accidental predicate visibility in the backwards-compatible wrapper for data flow configurations. In particular `DataFlow::hasFlowPath`, `DataFlow::hasFlow`, `DataFlow::hasFlowTo`, and `DataFlow::hasFlowToExpr` were accidentally exposed in a single version.
-
-## 0.6.1
-
-No user-facing changes.
-
-## 0.6.0
-
-### Breaking Changes
-
-* The `semmle.code.cpp.commons.Buffer` and `semmle.code.cpp.commons.NullTermination` libraries no longer expose `semmle.code.cpp.dataflow.DataFlow`. Please import `semmle.code.cpp.dataflow.DataFlow` directly.
-
-### Deprecated APIs
-
-* The `WriteConfig` taint tracking configuration has been deprecated. Please use `WriteFlow`.
-
-### New Features
-
-* Added support for merging two `PathGraph`s via disjoint union to allow results from multiple data flow computations in a single `path-problem` query.
-
-### Major Analysis Improvements
-
-* A new C/C++ dataflow library (`semmle.code.cpp.dataflow.new.DataFlow`) has been added.
-  The new library behaves much more like the dataflow library of other CodeQL supported
-  languages by following use-use dataflow paths instead of def-use dataflow paths.
-  The new library also better supports dataflow through indirections, and new predicates
-  such as `Node::asIndirectExpr` have been added to facilitate working with indirections.
-
-  The `semmle.code.cpp.ir.dataflow.DataFlow` library is now identical to the new
-  `semmle.code.cpp.dataflow.new.DataFlow` library.
-* The main data flow and taint tracking APIs have been changed. The old APIs
-  remain in place for now and translate to the new through a
-  backwards-compatible wrapper. If multiple configurations are in scope
-  simultaneously, then this may affect results slightly. The new API is quite
-  similar to the old, but makes use of a configuration module instead of a
-  configuration class.
-
-### Minor Analysis Improvements
-
-* Deleted the deprecated `hasGeneratedCopyConstructor` and `hasGeneratedCopyAssignmentOperator` predicates from the `Folder` class.
-* Deleted the deprecated `getPath` and `getFolder` predicates from the `XmlFile` class.
-* Deleted the deprecated `getMustlockFunction`, `getTrylockFunction`, `getLockFunction`, and `getUnlockFunction` predicates from the `MutexType` class.
-* Deleted the deprecated `getPosInBasicBlock` predicate from the `SubBasicBlock` class.
-* Deleted the deprecated `getExpr` predicate from the `PointerDereferenceExpr` class.
-* Deleted the deprecated `getUseInstruction` and `getDefinitionInstruction` predicates from the `Operand` class.
-* Deleted the deprecated `isInParameter`, `isInParameterPointer`, and `isInQualifier` predicates from the `FunctionInput` class.
-* Deleted the deprecated `isOutParameterPointer`, `isOutQualifier`, `isOutReturnValue`, and `isOutReturnPointer` predicate from the `FunctionOutput` class.
-* Deleted the deprecated 3-argument `isGuardPhi` predicate from the `RangeSsaDefinition` class.
-
-## 0.5.4
-
-No user-facing changes.
-
-## 0.5.3
-
-No user-facing changes.
-
-## 0.5.2
-
-No user-facing changes.
-
 ## 0.5.1
 
 No user-facing changes.

cpp/ql/lib/change-notes/2023-09-06-as-defining-argument-off-by-one-fix.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The `DataFlow::asDefiningArgument` predicate now takes its argument from the range starting at `1` instead of `2`. Queries that depend on the single-parameter version of `DataFlow::asDefiningArgument` should have their arguments updated accordingly.

cpp/ql/lib/change-notes/2023-09-07-return-from-end.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* Treat functions that reach the end of the function as returning in the IR.
-  They used to be treated as unreachable but it is allowed in C. 

cpp/ql/lib/change-notes/2023-09-08-more-unreachble.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* Functions that do not return due to calling functions that don't return (e.g. `exit`) are now detected as
- non-returning in the IR and dataflow.

cpp/ql/lib/change-notes/released/0.5.2.md

@@ -1,3 +0,0 @@
-## 0.5.2
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.5.3.md

@@ -1,3 +0,0 @@
-## 0.5.3
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.5.4.md

@@ -1,3 +0,0 @@
-## 0.5.4
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.6.0.md

@@ -1,42 +0,0 @@
-## 0.6.0
-
-### Breaking Changes
-
-* The `semmle.code.cpp.commons.Buffer` and `semmle.code.cpp.commons.NullTermination` libraries no longer expose `semmle.code.cpp.dataflow.DataFlow`. Please import `semmle.code.cpp.dataflow.DataFlow` directly.
-
-### Deprecated APIs
-
-* The `WriteConfig` taint tracking configuration has been deprecated. Please use `WriteFlow`.
-
-### New Features
-
-* Added support for merging two `PathGraph`s via disjoint union to allow results from multiple data flow computations in a single `path-problem` query.
-
-### Major Analysis Improvements
-
-* A new C/C++ dataflow library (`semmle.code.cpp.dataflow.new.DataFlow`) has been added.
-  The new library behaves much more like the dataflow library of other CodeQL supported
-  languages by following use-use dataflow paths instead of def-use dataflow paths.
-  The new library also better supports dataflow through indirections, and new predicates
-  such as `Node::asIndirectExpr` have been added to facilitate working with indirections.
-
-  The `semmle.code.cpp.ir.dataflow.DataFlow` library is now identical to the new
-  `semmle.code.cpp.dataflow.new.DataFlow` library.
-* The main data flow and taint tracking APIs have been changed. The old APIs
-  remain in place for now and translate to the new through a
-  backwards-compatible wrapper. If multiple configurations are in scope
-  simultaneously, then this may affect results slightly. The new API is quite
-  similar to the old, but makes use of a configuration module instead of a
-  configuration class.
-
-### Minor Analysis Improvements
-
-* Deleted the deprecated `hasGeneratedCopyConstructor` and `hasGeneratedCopyAssignmentOperator` predicates from the `Folder` class.
-* Deleted the deprecated `getPath` and `getFolder` predicates from the `XmlFile` class.
-* Deleted the deprecated `getMustlockFunction`, `getTrylockFunction`, `getLockFunction`, and `getUnlockFunction` predicates from the `MutexType` class.
-* Deleted the deprecated `getPosInBasicBlock` predicate from the `SubBasicBlock` class.
-* Deleted the deprecated `getExpr` predicate from the `PointerDereferenceExpr` class.
-* Deleted the deprecated `getUseInstruction` and `getDefinitionInstruction` predicates from the `Operand` class.
-* Deleted the deprecated `isInParameter`, `isInParameterPointer`, and `isInQualifier` predicates from the `FunctionInput` class.
-* Deleted the deprecated `isOutParameterPointer`, `isOutQualifier`, `isOutReturnValue`, and `isOutReturnPointer` predicate from the `FunctionOutput` class.
-* Deleted the deprecated 3-argument `isGuardPhi` predicate from the `RangeSsaDefinition` class.

cpp/ql/lib/change-notes/released/0.6.1.md

@@ -1,3 +0,0 @@
-## 0.6.1
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.7.0.md

@@ -1,25 +0,0 @@
-## 0.7.0
-
-### Breaking Changes
-
-* The internal `SsaConsistency` module has been moved from `SSAConstruction` to `SSAConsitency`, and the deprecated `SSAConsistency` module has been removed.
-
-### Deprecated APIs
-
-* The single-parameter predicates `ArrayOrVectorAggregateLiteral.getElementExpr` and `ClassAggregateLiteral.getFieldExpr` have been deprecated in favor of `ArrayOrVectorAggregateLiteral.getAnElementExpr` and `ClassAggregateLiteral.getAFieldExpr`.
-* The recently introduced new data flow and taint tracking APIs have had a
-  number of module and predicate renamings. The old APIs remain in place for
-  now.
-* The `SslContextCallAbstractConfig`, `SslContextCallConfig`, `SslContextCallBannedProtocolConfig`, `SslContextCallTls12ProtocolConfig`, `SslContextCallTls13ProtocolConfig`, `SslContextCallTlsProtocolConfig`, `SslContextFlowsToSetOptionConfig`, `SslOptionConfig` dataflow configurations from `BoostorgAsio` have been deprecated. Please use `SslContextCallConfigSig`, `SslContextCallGlobal`, `SslContextCallFlow`, `SslContextCallBannedProtocolFlow`, `SslContextCallTls12ProtocolFlow`, `SslContextCallTls13ProtocolFlow`, `SslContextCallTlsProtocolFlow`, `SslContextFlowsToSetOptionFlow`.
-
-### New Features
-
-* Added overridable predicates `getSizeExpr` and `getSizeMult` to the `BufferAccess` class (`semmle.code.cpp.security.BufferAccess.qll`). This makes it possible to model a larger class of buffer reads and writes using the library.
-
-### Minor Analysis Improvements
-
-* The `BufferAccess` library (`semmle.code.cpp.security.BufferAccess`) no longer matches buffer accesses inside unevaluated contexts (such as inside `sizeof` or `decltype` expressions). As a result, queries using this library may see fewer false positives.
-
-### Bug Fixes
-
-* Fixed some accidental predicate visibility in the backwards-compatible wrapper for data flow configurations. In particular `DataFlow::hasFlowPath`, `DataFlow::hasFlow`, `DataFlow::hasFlowTo`, and `DataFlow::hasFlowToExpr` were accidentally exposed in a single version.

cpp/ql/lib/change-notes/released/0.7.1.md

@@ -1,3 +0,0 @@
-## 0.7.1
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.7.2.md

@@ -1,15 +0,0 @@
-## 0.7.2
-
-### New Features
-
-* Added an AST-based interface (`semmle.code.cpp.rangeanalysis.new.RangeAnalysis`) for the relative range analysis library.
-* A new predicate `BarrierGuard::getAnIndirectBarrierNode` has been added to the new dataflow library (`semmle.code.cpp.dataflow.new.DataFlow`) to mark indirect expressions as barrier nodes using the `BarrierGuard` API.
-
-### Major Analysis Improvements
-
-* In the intermediate representation, handling of control flow after non-returning calls has been improved. This should remove false positives in queries that use the intermedite representation or libraries based on it, including the new data flow library.
-
-### Minor Analysis Improvements
-
-* The `StdNamespace` class now also includes all inline namespaces that are children of `std` namespace.
-* The new dataflow (`semmle.code.cpp.dataflow.new.DataFlow`) and taint-tracking libraries (`semmle.code.cpp.dataflow.new.TaintTracking`) now support tracking flow through static local variables.

cpp/ql/lib/change-notes/released/0.7.3.md

@@ -1,7 +0,0 @@
-## 0.7.3
-
-### Minor Analysis Improvements
-
-* Deleted the deprecated `hasCopyConstructor` predicate from the `Class` class in `Class.qll`.
-* Deleted many deprecated predicates and classes with uppercase `AST`, `SSA`, `CFG`, `API`, etc. in their names. Use the PascalCased versions instead.
-* Deleted the deprecated `CodeDuplication.qll` file.

cpp/ql/lib/change-notes/released/0.7.4.md

@@ -1,3 +0,0 @@
-## 0.7.4
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.8.0.md

@@ -1,9 +0,0 @@
-## 0.8.0
-
-### New Features
-
-* The `ProductFlow::StateConfigSig` signature now includes default predicates for `isBarrier1`, `isBarrier2`, `isAdditionalFlowStep1`, and `isAdditionalFlowStep1`. Hence, it is no longer needed to provide `none()` implementations of these predicates if they are not needed.
-
-### Minor Analysis Improvements
-
-* Deleted the deprecated `getURL` predicate from the `Container`, `Folder`, and `File` classes. Use the `getLocation` predicate instead.

cpp/ql/lib/change-notes/released/0.8.1.md

@@ -1,19 +0,0 @@
-## 0.8.1
-
-### Deprecated APIs
-
-* The library `semmle.code.cpp.dataflow.DataFlow` has been deprecated. Please use `semmle.code.cpp.dataflow.new.DataFlow` instead.
-
-### New Features
-
-* The `DataFlow::StateConfigSig` signature module has gained default implementations for `isBarrier/2` and `isAdditionalFlowStep/4`. 
-  Hence it is no longer needed to provide `none()` implementations of these predicates if they are not needed.
-
-### Minor Analysis Improvements
-
-* Data flow configurations can now include a predicate `neverSkip(Node node)`
-  in order to ensure inclusion of certain nodes in the path explanations. The
-  predicate defaults to the end-points of the additional flow steps provided in
-  the configuration, which means that such steps now always are visible by
-  default in path explanations.
-* The `IRGuards` library has improved handling of pointer addition and subtraction operations.

cpp/ql/lib/change-notes/released/0.9.0.md

@@ -1,14 +0,0 @@
-## 0.9.0
-
-### Breaking Changes
-
-* The `shouldPrintFunction` predicate from `PrintAstConfiguration` has been replaced by `shouldPrintDeclaration`. Users should now override `shouldPrintDeclaration` if they want to limit the declarations that should be printed.
-* The `shouldPrintFunction` predicate from `PrintIRConfiguration` has been replaced by `shouldPrintDeclaration`. Users should now override `shouldPrintDeclaration` if they want to limit the declarations that should be printed.
-
-### Major Analysis Improvements
-
-* The `PrintAST` library now also prints global and namespace variables and their initializers.
-
-### Minor Analysis Improvements
-
-* The `_Float128x` type is no longer exposed as a builtin type. As this type could not occur any code base, this should only affect queries that explicitly looked at the builtin types.

cpp/ql/lib/change-notes/released/0.9.1.md

@@ -1,3 +0,0 @@
-## 0.9.1
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.9.2.md

@@ -1,14 +0,0 @@
-## 0.9.2
-
-### Deprecated APIs
-
-* `getAllocatorCall` on `DeleteExpr` and `DeleteArrayExpr` has been deprecated. `getDeallocatorCall` should be used instead.
-
-### New Features
-
-* Added `DeleteOrDeleteArrayExpr` as a super type of `DeleteExpr` and `DeleteArrayExpr`
-
-### Minor Analysis Improvements
-
-* `delete` and `delete[]` are now modeled as calls to the relevant `operator delete` in the IR. In the case of a dynamic delete call a new instruction `VirtualDeleteFunctionAddress` is used to represent a function that dispatches to the correct delete implementation.
-* Only the 2 level indirection of `argv` (corresponding to `**argv`) is consided for `FlowSource`.

cpp/ql/lib/change-notes/released/0.9.3.md

@@ -1,3 +0,0 @@
-## 0.9.3
-
-No user-facing changes.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.9.3
+lastReleaseVersion: 0.5.1

cpp/ql/lib/experimental/semmle/code/cpp/dataflow/ProductFlow.qll

@@ -0,0 +1,301 @@
+import experimental.semmle.code.cpp.ir.dataflow.DataFlow
+import experimental.semmle.code.cpp.ir.dataflow.DataFlow2
+
+module ProductFlow {
+  abstract class Configuration extends string {
+    bindingset[this]
+    Configuration() { any() }
+
+    /**
+     * Holds if `(source1, source2)` is a relevant data flow source.
+     *
+     * `source1` and `source2` must belong to the same callable.
+     */
+    predicate isSourcePair(DataFlow::Node source1, DataFlow::Node source2) { none() }
+
+    /**
+     * Holds if `(source1, source2)` is a relevant data flow source with initial states `state1`
+     * and `state2`, respectively.
+     *
+     * `source1` and `source2` must belong to the same callable.
+     */
+    predicate isSourcePair(
+      DataFlow::Node source1, DataFlow::FlowState state1, DataFlow::Node source2,
+      DataFlow::FlowState state2
+    ) {
+      state1 = "" and
+      state2 = "" and
+      this.isSourcePair(source1, source2)
+    }
+
+    /**
+     * Holds if `(sink1, sink2)` is a relevant data flow sink.
+     *
+     * `sink1` and `sink2` must belong to the same callable.
+     */
+    predicate isSinkPair(DataFlow::Node sink1, DataFlow::Node sink2) { none() }
+
+    /**
+     * Holds if `(sink1, sink2)` is a relevant data flow sink with final states `state1`
+     * and `state2`, respectively.
+     *
+     * `sink1` and `sink2` must belong to the same callable.
+     */
+    predicate isSinkPair(
+      DataFlow::Node sink1, DataFlow::FlowState state1, DataFlow::Node sink2,
+      DataFlow::FlowState state2
+    ) {
+      state1 = "" and
+      state2 = "" and
+      this.isSinkPair(sink1, sink2)
+    }
+
+    /**
+     * Holds if data flow through `node` is prohibited through the first projection of the product
+     * dataflow graph when the flow state is `state`.
+     */
+    predicate isBarrier1(DataFlow::Node node, DataFlow::FlowState state) {
+      this.isBarrier1(node) and state = ""
+    }
+
+    /**
+     * Holds if data flow through `node` is prohibited through the second projection of the product
+     * dataflow graph when the flow state is `state`.
+     */
+    predicate isBarrier2(DataFlow::Node node, DataFlow::FlowState state) {
+      this.isBarrier2(node) and state = ""
+    }
+
+    /**
+     * Holds if data flow through `node` is prohibited through the first projection of the product
+     * dataflow graph.
+     */
+    predicate isBarrier1(DataFlow::Node node) { none() }
+
+    /**
+     * Holds if data flow through `node` is prohibited through the second projection of the product
+     * dataflow graph.
+     */
+    predicate isBarrier2(DataFlow::Node node) { none() }
+
+    /**
+     * Holds if data flow out of `node` is prohibited in the first projection of the product
+     * dataflow graph.
+     */
+    predicate isBarrierOut1(DataFlow::Node node) { none() }
+
+    /**
+     * Holds if data flow out of `node` is prohibited in the second projection of the product
+     * dataflow graph.
+     */
+    predicate isBarrierOut2(DataFlow::Node node) { none() }
+
+    /*
+     * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps in
+     * the first projection of the product dataflow graph.
+     */
+
+    predicate isAdditionalFlowStep1(DataFlow::Node node1, DataFlow::Node node2) { none() }
+
+    /**
+     * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps in
+     * the first projection of the product dataflow graph.
+     *
+     * This step is only applicable in `state1` and updates the flow state to `state2`.
+     */
+    predicate isAdditionalFlowStep1(
+      DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,
+      DataFlow::FlowState state2
+    ) {
+      state1 instanceof DataFlow::FlowStateEmpty and
+      state2 instanceof DataFlow::FlowStateEmpty and
+      this.isAdditionalFlowStep1(node1, node2)
+    }
+
+    /**
+     * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps in
+     * the second projection of the product dataflow graph.
+     */
+    predicate isAdditionalFlowStep2(DataFlow::Node node1, DataFlow::Node node2) { none() }
+
+    /**
+     * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps in
+     * the second projection of the product dataflow graph.
+     *
+     * This step is only applicable in `state1` and updates the flow state to `state2`.
+     */
+    predicate isAdditionalFlowStep2(
+      DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,
+      DataFlow::FlowState state2
+    ) {
+      state1 instanceof DataFlow::FlowStateEmpty and
+      state2 instanceof DataFlow::FlowStateEmpty and
+      this.isAdditionalFlowStep2(node1, node2)
+    }
+
+    /**
+     * Holds if data flow into `node` is prohibited in the first projection of the product
+     * dataflow graph.
+     */
+    predicate isBarrierIn1(DataFlow::Node node) { none() }
+
+    /**
+     * Holds if data flow into `node` is prohibited in the second projection of the product
+     * dataflow graph.
+     */
+    predicate isBarrierIn2(DataFlow::Node node) { none() }
+
+    predicate hasFlowPath(
+      DataFlow::PathNode source1, DataFlow2::PathNode source2, DataFlow::PathNode sink1,
+      DataFlow2::PathNode sink2
+    ) {
+      reachable(this, source1, source2, sink1, sink2)
+    }
+  }
+
+  private import Internal
+
+  module Internal {
+    class Conf1 extends DataFlow::Configuration {
+      Conf1() { this = "Conf1" }
+
+      override predicate isSource(DataFlow::Node source, DataFlow::FlowState state) {
+        exists(Configuration conf | conf.isSourcePair(source, state, _, _))
+      }
+
+      override predicate isSink(DataFlow::Node sink, DataFlow::FlowState state) {
+        exists(Configuration conf | conf.isSinkPair(sink, state, _, _))
+      }
+
+      override predicate isBarrier(DataFlow::Node node, DataFlow::FlowState state) {
+        exists(Configuration conf | conf.isBarrier1(node, state))
+      }
+
+      override predicate isBarrierOut(DataFlow::Node node) {
+        exists(Configuration conf | conf.isBarrierOut1(node))
+      }
+
+      override predicate isAdditionalFlowStep(
+        DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,
+        DataFlow::FlowState state2
+      ) {
+        exists(Configuration conf | conf.isAdditionalFlowStep1(node1, state1, node2, state2))
+      }
+
+      override predicate isBarrierIn(DataFlow::Node node) {
+        exists(Configuration conf | conf.isBarrierIn1(node))
+      }
+    }
+
+    class Conf2 extends DataFlow2::Configuration {
+      Conf2() { this = "Conf2" }
+
+      override predicate isSource(DataFlow::Node source, DataFlow::FlowState state) {
+        exists(Configuration conf, DataFlow::PathNode source1 |
+          conf.isSourcePair(source1.getNode(), source1.getState(), source, state) and
+          any(Conf1 c).hasFlowPath(source1, _)
+        )
+      }
+
+      override predicate isSink(DataFlow::Node sink, DataFlow::FlowState state) {
+        exists(Configuration conf, DataFlow::PathNode sink1 |
+          conf.isSinkPair(sink1.getNode(), sink1.getState(), sink, state) and
+          any(Conf1 c).hasFlowPath(_, sink1)
+        )
+      }
+
+      override predicate isBarrier(DataFlow::Node node, DataFlow::FlowState state) {
+        exists(Configuration conf | conf.isBarrier2(node, state))
+      }
+
+      override predicate isBarrierOut(DataFlow::Node node) {
+        exists(Configuration conf | conf.isBarrierOut2(node))
+      }
+
+      override predicate isAdditionalFlowStep(
+        DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,
+        DataFlow::FlowState state2
+      ) {
+        exists(Configuration conf | conf.isAdditionalFlowStep2(node1, state1, node2, state2))
+      }
+
+      override predicate isBarrierIn(DataFlow::Node node) {
+        exists(Configuration conf | conf.isBarrierIn2(node))
+      }
+    }
+  }
+
+  pragma[nomagic]
+  private predicate reachableInterprocEntry(
+    Configuration conf, DataFlow::PathNode source1, DataFlow2::PathNode source2,
+    DataFlow::PathNode node1, DataFlow2::PathNode node2
+  ) {
+    conf.isSourcePair(node1.getNode(), node1.getState(), node2.getNode(), node2.getState()) and
+    node1 = source1 and
+    node2 = source2
+    or
+    exists(
+      DataFlow::PathNode midEntry1, DataFlow2::PathNode midEntry2, DataFlow::PathNode midExit1,
+      DataFlow2::PathNode midExit2
+    |
+      reachableInterprocEntry(conf, source1, source2, midEntry1, midEntry2) and
+      interprocEdgePair(midExit1, midExit2, node1, node2) and
+      localPathStep1*(midEntry1, midExit1) and
+      localPathStep2*(midEntry2, midExit2)
+    )
+  }
+
+  private predicate localPathStep1(DataFlow::PathNode pred, DataFlow::PathNode succ) {
+    DataFlow::PathGraph::edges(pred, succ) and
+    pragma[only_bind_out](pred.getNode().getEnclosingCallable()) =
+      pragma[only_bind_out](succ.getNode().getEnclosingCallable())
+  }
+
+  private predicate localPathStep2(DataFlow2::PathNode pred, DataFlow2::PathNode succ) {
+    DataFlow2::PathGraph::edges(pred, succ) and
+    pragma[only_bind_out](pred.getNode().getEnclosingCallable()) =
+      pragma[only_bind_out](succ.getNode().getEnclosingCallable())
+  }
+
+  pragma[nomagic]
+  private predicate interprocEdge1(
+    Declaration predDecl, Declaration succDecl, DataFlow::PathNode pred1, DataFlow::PathNode succ1
+  ) {
+    DataFlow::PathGraph::edges(pred1, succ1) and
+    predDecl != succDecl and
+    pred1.getNode().getEnclosingCallable() = predDecl and
+    succ1.getNode().getEnclosingCallable() = succDecl
+  }
+
+  pragma[nomagic]
+  private predicate interprocEdge2(
+    Declaration predDecl, Declaration succDecl, DataFlow2::PathNode pred2, DataFlow2::PathNode succ2
+  ) {
+    DataFlow2::PathGraph::edges(pred2, succ2) and
+    predDecl != succDecl and
+    pred2.getNode().getEnclosingCallable() = predDecl and
+    succ2.getNode().getEnclosingCallable() = succDecl
+  }
+
+  private predicate interprocEdgePair(
+    DataFlow::PathNode pred1, DataFlow2::PathNode pred2, DataFlow::PathNode succ1,
+    DataFlow2::PathNode succ2
+  ) {
+    exists(Declaration predDecl, Declaration succDecl |
+      interprocEdge1(predDecl, succDecl, pred1, succ1) and
+      interprocEdge2(predDecl, succDecl, pred2, succ2)
+    )
+  }
+
+  private predicate reachable(
+    Configuration conf, DataFlow::PathNode source1, DataFlow2::PathNode source2,
+    DataFlow::PathNode sink1, DataFlow2::PathNode sink2
+  ) {
+    exists(DataFlow::PathNode mid1, DataFlow2::PathNode mid2 |
+      reachableInterprocEntry(conf, source1, source2, mid1, mid2) and
+      conf.isSinkPair(sink1.getNode(), sink1.getState(), sink2.getNode(), sink2.getState()) and
+      localPathStep1*(mid1, sink1) and
+      localPathStep2*(mid2, sink2)
+    )
+  }
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/DataFlow.qll

@@ -0,0 +1,26 @@
+/**
+ * Provides a library for local (intra-procedural) and global (inter-procedural)
+ * data flow analysis: deciding whether data can flow from a _source_ to a
+ * _sink_. This library differs from the one in `semmle.code.cpp.dataflow` in that
+ * this library uses the IR (Intermediate Representation) library, which provides
+ * a more precise semantic representation of the program, whereas the other dataflow
+ * library uses the more syntax-oriented ASTs. This library should provide more accurate
+ * results than the AST-based library in most scenarios.
+ *
+ * Unless configured otherwise, _flow_ means that the exact value of
+ * the source may reach the sink. We do not track flow across pointer
+ * dereferences or array indexing.
+ *
+ * To use global (interprocedural) data flow, extend the class
+ * `DataFlow::Configuration` as documented on that class. To use local
+ * (intraprocedural) data flow between expressions, call
+ * `DataFlow::localExprFlow`. For more general cases of local data flow, call
+ * `DataFlow::localFlow` or `DataFlow::localFlowStep` with arguments of type
+ * `DataFlow::Node`.
+ */
+
+import cpp
+
+module DataFlow {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/DataFlow2.qll

@@ -0,0 +1,16 @@
+/**
+ * Provides a `DataFlow2` module, which is a copy of the `DataFlow` module. Use
+ * this class when data-flow configurations must depend on each other. Two
+ * classes extending `DataFlow::Configuration` should never depend on each
+ * other, but one of them should instead depend on a
+ * `DataFlow2::Configuration`, a `DataFlow3::Configuration`, or a
+ * `DataFlow4::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.DataFlow` for the full documentation.
+ */
+
+import cpp
+
+module DataFlow2 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl2
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/DataFlow3.qll

@@ -0,0 +1,16 @@
+/**
+ * Provides a `DataFlow3` module, which is a copy of the `DataFlow` module. Use
+ * this class when data-flow configurations must depend on each other. Two
+ * classes extending `DataFlow::Configuration` should never depend on each
+ * other, but one of them should instead depend on a
+ * `DataFlow2::Configuration`, a `DataFlow3::Configuration`, or a
+ * `DataFlow4::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.DataFlow` for the full documentation.
+ */
+
+import cpp
+
+module DataFlow3 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl3
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/DataFlow4.qll

@@ -0,0 +1,16 @@
+/**
+ * Provides a `DataFlow4` module, which is a copy of the `DataFlow` module. Use
+ * this class when data-flow configurations must depend on each other. Two
+ * classes extending `DataFlow::Configuration` should never depend on each
+ * other, but one of them should instead depend on a
+ * `DataFlow2::Configuration`, a `DataFlow3::Configuration`, or a
+ * `DataFlow4::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.DataFlow` for the full documentation.
+ */
+
+import cpp
+
+module DataFlow4 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl4
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/ResolveCall.qll

@@ -0,0 +1,23 @@
+/**
+ * Provides a predicate for non-contextual virtual dispatch and function
+ * pointer resolution.
+ */
+
+import cpp
+private import semmle.code.cpp.ir.ValueNumbering
+private import internal.DataFlowDispatch
+private import semmle.code.cpp.ir.IR
+
+/**
+ * Resolve potential target function(s) for `call`.
+ *
+ * If `call` is a call through a function pointer (`ExprCall`) or its target is
+ * a virtual member function, simple data flow analysis is performed in order
+ * to identify the possible target(s).
+ */
+Function resolveCall(Call call) {
+  exists(CallInstruction callInstruction |
+    callInstruction.getAst() = call and
+    result = viableCallable(callInstruction)
+  )
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/TaintTracking.qll

@@ -0,0 +1,23 @@
+/**
+ * Provides classes for performing local (intra-procedural) and
+ * global (inter-procedural) taint-tracking analyses.
+ *
+ * We define _taint propagation_ informally to mean that a substantial part of
+ * the information from the source is preserved at the sink. For example, taint
+ * propagates from `x` to `x + 100`, but it does not propagate from `x` to `x >
+ * 100` since we consider a single bit of information to be too little.
+ *
+ * To use global (interprocedural) taint tracking, extend the class
+ * `TaintTracking::Configuration` as documented on that class. To use local
+ * (intraprocedural) taint tracking between expressions, call
+ * `TaintTracking::localExprTaint`. For more general cases of local taint
+ * tracking, call `TaintTracking::localTaint` or
+ * `TaintTracking::localTaintStep` with arguments of type `DataFlow::Node`.
+ */
+
+import semmle.code.cpp.ir.dataflow.DataFlow
+import semmle.code.cpp.ir.dataflow.DataFlow2
+
+module TaintTracking {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.tainttracking1.TaintTrackingImpl
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/TaintTracking2.qll

@@ -0,0 +1,15 @@
+/**
+ * Provides a `TaintTracking2` module, which is a copy of the `TaintTracking`
+ * module. Use this class when data-flow configurations or taint-tracking
+ * configurations must depend on each other. Two classes extending
+ * `DataFlow::Configuration` should never depend on each other, but one of them
+ * should instead depend on a `DataFlow2::Configuration`, a
+ * `DataFlow3::Configuration`, or a `DataFlow4::Configuration`. The
+ * `TaintTracking::Configuration` class extends `DataFlow::Configuration`, and
+ * `TaintTracking2::Configuration` extends `DataFlow2::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.TaintTracking` for the full documentation.
+ */
+module TaintTracking2 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.tainttracking2.TaintTrackingImpl
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/TaintTracking3.qll

@@ -0,0 +1,15 @@
+/**
+ * Provides a `TaintTracking3` module, which is a copy of the `TaintTracking`
+ * module. Use this class when data-flow configurations or taint-tracking
+ * configurations must depend on each other. Two classes extending
+ * `DataFlow::Configuration` should never depend on each other, but one of them
+ * should instead depend on a `DataFlow2::Configuration`, a
+ * `DataFlow3::Configuration`, or a `DataFlow4::Configuration`. The
+ * `TaintTracking::Configuration` class extends `DataFlow::Configuration`, and
+ * `TaintTracking2::Configuration` extends `DataFlow2::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.TaintTracking` for the full documentation.
+ */
+module TaintTracking3 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.tainttracking3.TaintTrackingImpl
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll

@@ -0,0 +1,273 @@
+private import cpp
+private import semmle.code.cpp.ir.IR
+private import experimental.semmle.code.cpp.ir.dataflow.DataFlow
+private import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowPrivate
+private import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowUtil
+private import DataFlowImplCommon as DataFlowImplCommon
+
+/**
+ * Gets a function that might be called by `call`.
+ */
+cached
+Function viableCallable(CallInstruction call) {
+  DataFlowImplCommon::forceCachingInSameStage() and
+  result = call.getStaticCallTarget()
+  or
+  // If the target of the call does not have a body in the snapshot, it might
+  // be because the target is just a header declaration, and the real target
+  // will be determined at run time when the caller and callee are linked
+  // together by the operating system's dynamic linker. In case a _unique_
+  // function with the right signature is present in the database, we return
+  // that as a potential callee.
+  exists(string qualifiedName, int nparams |
+    callSignatureWithoutBody(qualifiedName, nparams, call) and
+    functionSignatureWithBody(qualifiedName, nparams, result) and
+    strictcount(Function other | functionSignatureWithBody(qualifiedName, nparams, other)) = 1
+  )
+  or
+  // Virtual dispatch
+  result = call.(VirtualDispatch::DataSensitiveCall).resolve()
+}
+
+/**
+ * Provides virtual dispatch support compatible with the original
+ * implementation of `semmle.code.cpp.security.TaintTracking`.
+ */
+private module VirtualDispatch {
+  /** A call that may dispatch differently depending on the qualifier value. */
+  abstract class DataSensitiveCall extends DataFlowCall {
+    /**
+     * Gets the node whose value determines the target of this call. This node
+     * could be the qualifier of a virtual dispatch or the function-pointer
+     * expression in a call to a function pointer. What they have in common is
+     * that we need to find out which data flows there, and then it's up to the
+     * `resolve` predicate to stitch that information together and resolve the
+     * call.
+     */
+    abstract DataFlow::Node getDispatchValue();
+
+    /** Gets a candidate target for this call. */
+    abstract Function resolve();
+
+    /**
+     * Whether `src` can flow to this call.
+     *
+     * Searches backwards from `getDispatchValue()` to `src`. The `allowFromArg`
+     * parameter is true when the search is allowed to continue backwards into
+     * a parameter; non-recursive callers should pass `_` for `allowFromArg`.
+     */
+    predicate flowsFrom(DataFlow::Node src, boolean allowFromArg) {
+      src = this.getDispatchValue() and allowFromArg = true
+      or
+      exists(DataFlow::Node other, boolean allowOtherFromArg |
+        this.flowsFrom(other, allowOtherFromArg)
+      |
+        // Call argument
+        exists(DataFlowCall call, Position i |
+          other
+              .(DataFlow::ParameterNode)
+              .isParameterOf(pragma[only_bind_into](call).getStaticCallTarget(), i) and
+          src.(ArgumentNode).argumentOf(call, pragma[only_bind_into](pragma[only_bind_out](i)))
+        ) and
+        allowOtherFromArg = true and
+        allowFromArg = true
+        or
+        // Call return
+        exists(DataFlowCall call, ReturnKind returnKind |
+          other = getAnOutNode(call, returnKind) and
+          returnNodeWithKindAndEnclosingCallable(src, returnKind, call.getStaticCallTarget())
+        ) and
+        allowFromArg = false
+        or
+        // Local flow
+        DataFlow::localFlowStep(src, other) and
+        allowFromArg = allowOtherFromArg
+        or
+        // Flow from global variable to load.
+        exists(LoadInstruction load, GlobalOrNamespaceVariable var |
+          var = src.asVariable() and
+          other.asInstruction() = load and
+          addressOfGlobal(load.getSourceAddress(), var) and
+          // The `allowFromArg` concept doesn't play a role when `src` is a
+          // global variable, so we just set it to a single arbitrary value for
+          // performance.
+          allowFromArg = true
+        )
+        or
+        // Flow from store to global variable.
+        exists(StoreInstruction store, GlobalOrNamespaceVariable var |
+          var = other.asVariable() and
+          store = src.asInstruction() and
+          storeIntoGlobal(store, var) and
+          // Setting `allowFromArg` to `true` like in the base case means we
+          // treat a store to a global variable like the dispatch itself: flow
+          // may come from anywhere.
+          allowFromArg = true
+        )
+      )
+    }
+  }
+
+  pragma[noinline]
+  private predicate storeIntoGlobal(StoreInstruction store, GlobalOrNamespaceVariable var) {
+    addressOfGlobal(store.getDestinationAddress(), var)
+  }
+
+  /** Holds if `addressInstr` is an instruction that produces the address of `var`. */
+  private predicate addressOfGlobal(Instruction addressInstr, GlobalOrNamespaceVariable var) {
+    // Access directly to the global variable
+    addressInstr.(VariableAddressInstruction).getAstVariable() = var
+    or
+    // Access to a field on a global union
+    exists(FieldAddressInstruction fa |
+      fa = addressInstr and
+      fa.getObjectAddress().(VariableAddressInstruction).getAstVariable() = var and
+      fa.getField().getDeclaringType() instanceof Union
+    )
+  }
+
+  /**
+   * A ReturnNode with its ReturnKind and its enclosing callable.
+   *
+   * Used to fix a join ordering issue in flowsFrom.
+   */
+  pragma[noinline]
+  private predicate returnNodeWithKindAndEnclosingCallable(
+    ReturnNode node, ReturnKind kind, DataFlowCallable callable
+  ) {
+    node.getKind() = kind and
+    node.getEnclosingCallable() = callable
+  }
+
+  /** Call through a function pointer. */
+  private class DataSensitiveExprCall extends DataSensitiveCall {
+    DataSensitiveExprCall() { not exists(this.getStaticCallTarget()) }
+
+    override DataFlow::Node getDispatchValue() { result.asInstruction() = this.getCallTarget() }
+
+    override Function resolve() {
+      exists(FunctionInstruction fi |
+        this.flowsFrom(DataFlow::instructionNode(fi), _) and
+        result = fi.getFunctionSymbol()
+      ) and
+      (
+        this.getNumberOfArguments() <= result.getEffectiveNumberOfParameters() and
+        this.getNumberOfArguments() >= result.getEffectiveNumberOfParameters()
+        or
+        result.isVarargs()
+      )
+    }
+  }
+
+  /** Call to a virtual function. */
+  private class DataSensitiveOverriddenFunctionCall extends DataSensitiveCall {
+    DataSensitiveOverriddenFunctionCall() {
+      exists(this.getStaticCallTarget().(VirtualFunction).getAnOverridingFunction())
+    }
+
+    override DataFlow::Node getDispatchValue() { result.asInstruction() = this.getThisArgument() }
+
+    override MemberFunction resolve() {
+      exists(Class overridingClass |
+        this.overrideMayAffectCall(overridingClass, result) and
+        this.hasFlowFromCastFrom(overridingClass)
+      )
+    }
+
+    /**
+     * Holds if `this` is a virtual function call whose static target is
+     * overridden by `overridingFunction` in `overridingClass`.
+     */
+    pragma[noinline]
+    private predicate overrideMayAffectCall(Class overridingClass, MemberFunction overridingFunction) {
+      overridingFunction.getAnOverriddenFunction+() = this.getStaticCallTarget().(VirtualFunction) and
+      overridingFunction.getDeclaringType() = overridingClass
+    }
+
+    /**
+     * Holds if the qualifier of `this` has flow from an upcast from
+     * `derivedClass`.
+     */
+    pragma[noinline]
+    private predicate hasFlowFromCastFrom(Class derivedClass) {
+      exists(ConvertToBaseInstruction toBase |
+        this.flowsFrom(DataFlow::instructionNode(toBase), _) and
+        derivedClass = toBase.getDerivedClass()
+      )
+    }
+  }
+}
+
+/**
+ * Holds if `f` is a function with a body that has name `qualifiedName` and
+ * `nparams` parameter count. See `functionSignature`.
+ */
+private predicate functionSignatureWithBody(string qualifiedName, int nparams, Function f) {
+  functionSignature(f, qualifiedName, nparams) and
+  exists(f.getBlock())
+}
+
+/**
+ * Holds if the target of `call` is a function _with no definition_ that has
+ * name `qualifiedName` and `nparams` parameter count. See `functionSignature`.
+ */
+pragma[noinline]
+private predicate callSignatureWithoutBody(string qualifiedName, int nparams, CallInstruction call) {
+  exists(Function target |
+    target = call.getStaticCallTarget() and
+    not exists(target.getBlock()) and
+    functionSignature(target, qualifiedName, nparams)
+  )
+}
+
+/**
+ * Holds if `f` has name `qualifiedName` and `nparams` parameter count. This is
+ * an approximation of its signature for the purpose of matching functions that
+ * might be the same across link targets.
+ */
+private predicate functionSignature(Function f, string qualifiedName, int nparams) {
+  qualifiedName = f.getQualifiedName() and
+  nparams = f.getNumberOfParameters() and
+  not f.isStatic()
+}
+
+/**
+ * Holds if the set of viable implementations that can be called by `call`
+ * might be improved by knowing the call context.
+ */
+predicate mayBenefitFromCallContext(CallInstruction call, Function f) {
+  mayBenefitFromCallContext(call, f, _)
+}
+
+/**
+ * Holds if `call` is a call through a function pointer, and the pointer
+ * value is given as the `arg`'th argument to `f`.
+ */
+private predicate mayBenefitFromCallContext(
+  VirtualDispatch::DataSensitiveCall call, Function f, int arg
+) {
+  f = pragma[only_bind_out](call).getEnclosingCallable() and
+  exists(InitializeParameterInstruction init |
+    not exists(call.getStaticCallTarget()) and
+    init.getEnclosingFunction() = f and
+    call.flowsFrom(DataFlow::instructionNode(init), _) and
+    init.getParameter().getIndex() = arg
+  )
+}
+
+/**
+ * Gets a viable dispatch target of `call` in the context `ctx`. This is
+ * restricted to those `call`s for which a context might make a difference.
+ */
+Function viableImplInCallContext(CallInstruction call, CallInstruction ctx) {
+  result = viableCallable(call) and
+  exists(int i, Function f |
+    mayBenefitFromCallContext(pragma[only_bind_into](call), f, i) and
+    f = ctx.getStaticCallTarget() and
+    result = ctx.getArgument(i).getUnconvertedResultExpression().(FunctionAccess).getTarget()
+  )
+}
+
+/** Holds if arguments at position `apos` match parameters at position `ppos`. */
+pragma[inline]
+predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) { ppos = apos }