Ensure SetFormatterAssignment lhs/rhs match up

Introduce abstract `SanitizingFormatter` class as a base for all
sanitizing formatters

.bazelrc

@@ -1,9 +1,3 @@
-common --enable_platform_specific_config
-
-build --repo_env=CC=clang --repo_env=CXX=clang++
-
-build:linux --cxxopt=-std=c++20
-build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
-build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
+build --repo_env=CC=clang --repo_env=CXX=clang++ --cxxopt="-std=c++17"
 
 try-import %workspace%/local.bazelrc

.bazelversion

@@ -1 +1 @@
-6.3.1
+5.0.0

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "extensions": [
-    "rust-lang.rust-analyzer",
+    "rust-lang.rust",
     "bungcip.better-toml",
     "github.vscode-codeql",
     "hbenl.vscode-test-explorer",

.git-blame-ignore-revs

@@ -1,21 +0,0 @@
-# .git-blame-ignore-revs
-# Auto-formatted Java
-730eae952139209fe9fdf598541d608f4c0c0c84
-# Auto-formatted C#
-5ad7ed49dd3de03ec6dcfcb6848758a6a987e11c
-# Auto-formatted C/C++
-ef97e539ec1971494d4bba5cafe82e00bc8217ac
-# Auto-formatted Python
-21d5fa836b3a7d020ba45e8b8168b145a9772131
-# Auto-formatted JavaScript
-8d97fe9ed327a9546ff2eaf515cf0f5214deddd9
-# Auto-formatted Ruby
-a5d229903d2f12d45f2c2c38822f1d0e7504ae7f
-# Auto-formatted Go
-08c658e66bf867090033ea096e244a93d46c0aa7
-# Auto-formatted Swift
-711d7057f79fb7d72fc3b35e010bd018f9009169
-# Auto-formatted shared ql packs
-3640b6d3a8ce9edf8e1d3ed106fe8526cf255bc0
-# Auto-formatted taint tracking files
-159d8e978c51959b380838c080d891b66e763b19

.github/actions/cache-query-compilation/action.yml

@@ -9,7 +9,7 @@ inputs:
 outputs:
   cache-dir:
     description: "The directory where the cache was stored"
-    value: ${{ steps.output-compilation-dir.outputs.compdir }}
+    value: ${{ steps.fill-compilation-dir.outputs.compdir }}
 
 runs:
   using: composite
@@ -27,9 +27,7 @@ runs:
       if: ${{ github.event_name == 'pull_request' }}
       uses: actions/cache/restore@v3
       with:
-        path: |
-          **/.cache
-          ~/.codeql/compile-cache
+        path: '**/.cache'
         key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }}
         restore-keys: |
           codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
@@ -39,111 +37,18 @@ runs:
       if: ${{ github.event_name != 'pull_request' }}
       uses: actions/cache@v3
       with:
-        path: |
-          **/.cache
-          ~/.codeql/compile-cache
+        path: '**/.cache'
         key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
         restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
           codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
           codeql-compile-${{ inputs.key }}-main-
-    - name: Output-compilationdir
-      id: output-compilation-dir
+    - name: Fill compilation cache directory
+      id: fill-compilation-dir
       shell: bash
       run: |
+        # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+        node $GITHUB_WORKSPACE/.github/actions/cache-query-compilation/move-caches.js ${COMBINED_CACHE_DIR}
+
         echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
       env:
         COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
-    - name: Fill compilation cache directory
-      id: fill-compilation-dir
-      uses: actions/github-script@v6
-      env: 
-        COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
-      with:
-        script: |
-          // # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-          // mkdir -p ${COMBINED_CACHE_DIR}
-          // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-          // # copy the contents of the .cache folders into the combined cache folder.
-          // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-          // # clean up the .cache folders
-          // rm -rf **/.cache/*
-
-          const fs = require("fs");
-          const path = require("path");
-          const os = require("os");
-
-          // the first argv is the cache folder to create.
-          const COMBINED_CACHE_DIR = process.env.COMBINED_CACHE_DIR;
-
-          function* walkCaches(dir) {
-            const files = fs.readdirSync(dir, { withFileTypes: true });
-            for (const file of files) {
-              if (file.isDirectory()) {
-                const filePath = path.join(dir, file.name);
-                yield* walkCaches(filePath);
-                if (file.name === ".cache") {
-                  yield filePath;
-                }
-              }
-            }
-          }
-
-          async function copyDir(src, dest) {
-            for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
-              const srcPath = path.join(src, file.name);
-              const destPath = path.join(dest, file.name);
-              if (file.isDirectory()) {
-                if (!fs.existsSync(destPath)) {
-                  fs.mkdirSync(destPath);
-                }
-                await copyDir(srcPath, destPath);
-              } else {
-                await fs.promises.copyFile(srcPath, destPath);
-              }
-            }
-          }
-
-          async function main() {
-            const cacheDirs = [...walkCaches(".")];
-
-            for (const dir of cacheDirs) {
-              console.log(`Found .cache dir at ${dir}`);
-            }
-
-            const globalCacheDir = path.join(os.homedir(), ".codeql", "compile-cache");
-            if (fs.existsSync(globalCacheDir)) {
-              console.log("Found global home dir: " + globalCacheDir);
-              cacheDirs.push(globalCacheDir);
-            }
-
-            if (cacheDirs.length === 0) {
-              console.log("No cache dirs found");
-              return;
-            }
-
-            // mkdir -p ${COMBINED_CACHE_DIR}
-            fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
-
-            // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-            await Promise.all(
-              cacheDirs.map((cacheDir) =>
-                (async function () {
-                  await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
-                  await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
-                })()
-              )
-            );
-
-            // # copy the contents of the .cache folders into the combined cache folder.
-            // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-            await Promise.all(
-              cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
-            );
-
-            // # clean up the .cache folders
-            // rm -rf **/.cache/*
-            await Promise.all(
-              cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
-            );
-          }
-          main();

.github/actions/cache-query-compilation/move-caches.js

@@ -0,0 +1,75 @@
+// # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+// mkdir -p ${COMBINED_CACHE_DIR}
+// rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+// # copy the contents of the .cache folders into the combined cache folder.
+// cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+// # clean up the .cache folders
+// rm -rf **/.cache/*
+
+const fs = require("fs");
+const path = require("path");
+
+// the first argv is the cache folder to create.
+const COMBINED_CACHE_DIR = process.argv[2];
+
+function* walkCaches(dir) {
+  const files = fs.readdirSync(dir, { withFileTypes: true });
+  for (const file of files) {
+    if (file.isDirectory()) {
+      const filePath = path.join(dir, file.name);
+      yield* walkCaches(filePath);
+      if (file.name === ".cache") {
+        yield filePath;
+      }
+    }
+  }
+}
+
+async function copyDir(src, dest) {
+  for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
+    const srcPath = path.join(src, file.name);
+    const destPath = path.join(dest, file.name);
+    if (file.isDirectory()) {
+      if (!fs.existsSync(destPath)) {
+        fs.mkdirSync(destPath);
+      }
+      await copyDir(srcPath, destPath);
+    } else {
+      await fs.promises.copyFile(srcPath, destPath);
+    }
+  }
+}
+
+async function main() {
+  const cacheDirs = [...walkCaches(".")];
+
+  for (const dir of cacheDirs) {
+    console.log(`Found .cache dir at ${dir}`);
+  }
+
+  // mkdir -p ${COMBINED_CACHE_DIR}
+  fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
+
+  // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+  await Promise.all(
+    cacheDirs.map((cacheDir) =>
+      (async function () {
+        await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
+        await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
+      })()
+    )
+  );
+
+  // # copy the contents of the .cache folders into the combined cache folder.
+  // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+  await Promise.all(
+    cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
+  );
+
+  // # clean up the .cache folders
+  // rm -rf **/.cache/*
+  await Promise.all(
+    cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
+  );
+}
+main();

.github/actions/fetch-codeql/action.yml

@@ -19,6 +19,4 @@ runs:
         gh extension install github/gh-codeql
         gh codeql set-channel "$CHANNEL"
         gh codeql version
-        printf "CODEQL_FETCHED_CODEQL_PATH=" >> "${GITHUB_ENV}"
-        gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_ENV}"
         gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_PATH}"

.github/actions/find-latest-bundle/action.yml

@@ -0,0 +1,26 @@
+name: Find Latest CodeQL Bundle
+description: Finds the URL of the latest released version of the CodeQL bundle.
+outputs:
+  url:
+    description: The download URL of the latest CodeQL bundle release
+    value: ${{ steps.find-latest.outputs.url }}
+runs:
+  using: composite
+  steps:
+    - name: Find Latest Release
+      id: find-latest
+      shell: pwsh
+      run: |
+        $Latest = gh release list --repo github/codeql-action --exclude-drafts --limit 1000 |
+          ForEach-Object { $C = $_ -split "`t"; return @{ type = $C[1]; tag = $C[2]; } } |
+          Where-Object { $_.type -eq 'Latest' }
+
+        $Tag = $Latest.tag
+        if ($Tag -eq '') {
+          throw 'Failed to find latest bundle release.'
+        }
+
+        Write-Output "Latest bundle tag is '${Tag}'."
+        "url=https://github.com/github/codeql-action/releases/download/${Tag}/codeql-bundle-linux64.tar.gz" >> $env:GITHUB_OUTPUT
+      env:
+        GITHUB_TOKEN: ${{ github.token }}

.github/dependabot.yml

@@ -1,12 +1,19 @@
 version: 2
 updates:
   - package-ecosystem: "cargo"
-    directory: "ruby"
+    directory: "ruby/node-types"
     schedule:
       interval: "daily"
-
   - package-ecosystem: "cargo"
-    directory: "ql"
+    directory: "ruby/generator"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "cargo"
+    directory: "ruby/extractor"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "cargo"
+    directory: "ruby/autobuilder"
     schedule:
       interval: "daily"
 

.github/labeler.yml

@@ -11,7 +11,7 @@ Go:
   - change-notes/**/*go.*
 
 Java:
-  - any: [ 'java/**/*', '!java/kotlin-extractor/**/*', '!java/ql/test/kotlin/**/*' ]
+  - any: [ 'java/**/*', '!java/kotlin-extractor/**/*', '!java/kotlin-explorer/**/*', '!java/ql/test/kotlin/**/*' ]
   - change-notes/**/*java.*
 
 JS:
@@ -20,6 +20,7 @@ JS:
 
 Kotlin:
   - java/kotlin-extractor/**/*
+  - java/kotlin-explorer/**/*
   - java/ql/test/kotlin/**/*
 
 Python:
@@ -45,7 +46,11 @@ documentation:
 
 # Since these are all shared files that need to be synced, just pick _one_ copy of each.
 "DataFlow Library":
-  - "shared/dataflow/**/*"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"
 
 "ATM":
   - javascript/ql/experimental/adaptivethreatmodeling/**/*

.github/workflows/atm-check-query-suite.yml

@@ -0,0 +1,102 @@
+name: "ATM - Check query suite"
+
+env:
+  QUERY_PACK: javascript/ql/experimental/adaptivethreatmodeling/src
+  QUERY_SUITE: codeql-suites/javascript-atm-code-scanning.qls
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/atm-check-query-suite.yml"
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+  workflow_dispatch:
+
+jobs:
+  atm-check-query-suite:
+    runs-on: ubuntu-latest-xl
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup CodeQL
+        uses: ./.github/actions/fetch-codeql
+        with:
+          channel: release
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: atm-suite
+
+      - name: Install ATM model
+        run: |
+          set -exu
+
+          # Install dependencies of ATM query pack, i.e. the ATM model
+          codeql pack install "${QUERY_PACK}"
+
+          # Retrieve model checksum
+          model_checksum=$(codeql resolve extensions "${QUERY_PACK}/${QUERY_SUITE}" | jq -r '.models[0].checksum')
+
+          # Trust the model so that we can use it in the ATM boosted queries
+          mkdir -p "$HOME/.config/codeql"
+          echo "--insecurely-execute-ml-model-checksums ${model_checksum}" >> "$HOME/.config/codeql/config"
+
+      - name: Create test DB
+        run: |
+          DB_PATH="${RUNNER_TEMP}/db"
+          echo "DB_PATH=${DB_PATH}" >> "${GITHUB_ENV}"
+
+          codeql database create "${DB_PATH}" --source-root config/atm --language javascript 
+
+      - name: Run ATM query suite
+        run: |
+          SARIF_PATH="${RUNNER_TEMP}/sarif.json"
+          echo "SARIF_PATH=${SARIF_PATH}" >> "${GITHUB_ENV}"
+
+          codeql database analyze \
+            --threads=0 \
+            --ram 50000 \
+            --format sarif-latest \
+            --output "${SARIF_PATH}" \
+            --sarif-group-rules-by-pack \
+            -vv \
+            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
+            -- \
+            "${DB_PATH}" \
+            "${QUERY_PACK}/${QUERY_SUITE}"
+
+      - name: Upload SARIF
+        uses: actions/upload-artifact@v3
+        with:
+          name: javascript-ml-powered-queries.sarif
+          path: "${{ env.SARIF_PATH }}"
+          retention-days: 5
+
+      - name: Check results
+        run: |
+          # We should run at least the ML-powered queries in `expected_rules`.
+          expected_rules="js/ml-powered/nosql-injection js/ml-powered/path-injection js/ml-powered/sql-injection js/ml-powered/xss"
+
+          for rule in ${expected_rules}; do
+            found_rule=$(jq --arg rule "${rule}" '[.runs[0].tool.extensions[].rules | select(. != null) |
+              flatten | .[].id] | any(. == $rule)' "${SARIF_PATH}")
+            if [[ "${found_rule}" != "true" ]]; then
+              echo "Expected SARIF output to contain rule '${rule}', but found no such rule."
+              exit 1
+            else
+              echo "Found rule '${rule}'."
+            fi
+          done
+
+          # We should have at least one alert from an ML-powered query.
+          num_alerts=$(jq '[.runs[0].results[] |
+            select(.properties.score != null and (.rule.id | startswith("js/ml-powered/")))] | length' \
+            "${SARIF_PATH}")
+          if [[ "${num_alerts}" -eq 0 ]]; then
+            echo "Expected to find at least one alert from an ML-powered query but found ${num_alerts}."
+            exit 1
+          else
+            echo "Found ${num_alerts} alerts from ML-powered queries.";
+          fi

.github/workflows/atm-model-integration-tests.yml

@@ -0,0 +1,12 @@
+name: ATM Model Integration Tests
+
+on:
+  workflow_dispatch:
+
+jobs:
+  hello-world:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: foo
+        run: echo "Hello world"

.github/workflows/check-change-note.yml

@@ -8,9 +8,9 @@ on:
       - "*/ql/src/**/*.qll"
       - "*/ql/lib/**/*.ql"
       - "*/ql/lib/**/*.qll"
-      - "*/ql/lib/**/*.yml"
       - "!**/experimental/**"
       - "!ql/**"
+      - "!swift/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:
@@ -26,9 +26,9 @@ jobs:
         run: |
           gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' |
           grep true -c
-      - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md', 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text, or released/x.y.z.md for released change-notes
+      - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md' or 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text.
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$") or test("/change-notes/released/[0-9]*[.][0-9]*[.][0-9]*[.]md$"))' |
+          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$"))' |
           grep true -c

.github/workflows/check-implicit-this.yml

@@ -1,29 +0,0 @@
-name: "Check implicit this warnings"
-
-on:
-  workflow_dispatch:
-  pull_request:
-    paths:
-      - "**qlpack.yml"
-    branches:
-      - main
-      - "rc/*"
-
-jobs:
-  check:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Check that implicit this warnings is enabled for all packs
-        shell: bash
-        run: |
-          EXIT_CODE=0
-          packs="$(find . -iname 'qlpack.yml')"
-          for pack_file in ${packs}; do
-            option="$(yq '.warnOnImplicitThis' ${pack_file})"
-            if [ "${option}" != "true" ]; then
-              echo "::error file=${pack_file}::warnOnImplicitThis property must be set to 'true' for pack ${pack_file}"
-              EXIT_CODE=1
-            fi
-          done
-          exit "${EXIT_CODE}"

.github/workflows/check-qldoc.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 2
 
@@ -26,8 +26,9 @@ jobs:
         shell: bash
         run: |
           EXIT_CODE=0
+          # TODO: remove the swift exception from the regex when we fix generated QLdoc
           # TODO: remove the shared exception from the regex when coverage of qlpacks without dbschemes is supported
-          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!(shared))[a-z]*/ql/lib' || true; } | sort -u)"
+          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!(swift|shared))[a-z]*/ql/lib' || true; } | sort -u)"
           for pack_dir in ${changed_lib_packs}; do
             lang="${pack_dir%/ql/lib}"
             codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"

.github/workflows/check-query-ids.yml

@@ -16,6 +16,6 @@ jobs:
     name: Check query IDs
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Check for duplicate query IDs
         run: python3 misc/scripts/check-query-ids.py

.github/workflows/close-stale.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/stale@v8
+    - uses: actions/stale@v7
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Comment or remove the `Stale` label in order to avoid having this issue closed in 7 days.'

.github/workflows/codeql-analysis.yml

@@ -28,12 +28,12 @@ jobs:
 
     steps:
     - name: Setup dotnet
-      uses: actions/setup-dotnet@v3
+      uses: actions/setup-dotnet@v2
       with:
-        dotnet-version: 7.0.102
+        dotnet-version: 6.0.202
 
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/compile-queries.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest-xl
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
         with:
@@ -24,14 +24,14 @@ jobs:
         with: 
           key: all-queries
       - name: check formatting
-        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 -n 3000 -P 10 codeql query format -q --check-only
+        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
       - name: compile queries - check-only
         # run with --check-only if running in a PR (github.sha != main)
         if : ${{ github.event_name == 'pull_request' }}
         shell: bash
-        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
       - name: compile queries - full
         # do full compile if running on main - this populates the cache
         if : ${{ github.event_name != 'pull_request' }}
         shell: bash
-        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/csharp-qltest.yml

@@ -29,7 +29,7 @@ jobs:
   qlupgrade:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - name: Check DB upgrade scripts
         run: |
@@ -52,7 +52,8 @@ jobs:
       matrix:
         slice: ["1/2", "2/2"]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
       - uses: ./csharp/actions/create-extractor-pack
       - name: Cache compilation cache
         id: query-cache
@@ -61,41 +62,25 @@ jobs:
           key: csharp-qltest-${{ matrix.slice }}
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          CODEQL_PATH=$(gh codeql version --format=json | jq -r .unpackedLocation)
+          # The legacy ASP extractor is not in this repo, so take the one from the nightly build
+          mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
+          # Safe guard against using the bundled extractor
+          rm -rf "$CODEQL_PATH/csharp"
+          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}
   unit-tests:
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-2019]
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Setup dotnet
         uses: actions/setup-dotnet@v3
         with:
-          dotnet-version: 7.0.102
+          dotnet-version: 6.0.202
       - name: Extractor unit tests
         run: |
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 extractor/Semmle.Util.Tests
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 extractor/Semmle.Extraction.Tests
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 autobuilder/Semmle.Autobuild.CSharp.Tests
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"
-        shell: bash
-  stubgentest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./csharp/actions/create-extractor-pack
-      - name: Run stub generator tests
-        run: |
-          # Generate (Asp)NetCore stubs
-          STUBS_PATH=stubs_output
-          python3 ql/src/Stubs/make_stubs_nuget.py webapp Swashbuckle.AspNetCore.Swagger latest "$STUBS_PATH"
-          rm -rf ql/test/resources/stubs/_frameworks
-          # Update existing stubs in the repo with the freshly generated ones
-          mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/
-          git status
-          codeql test run --threads=0 --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
+          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Util.Tests"
+          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Extraction.Tests"
+          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests"
+          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"

.github/workflows/csv-coverage-metrics.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
       - name: Create empty database
@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
       - name: Create empty database

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -10,7 +10,6 @@ on:
       - "*/ql/src/**/*.qll"
       - "*/ql/lib/**/*.ql"
       - "*/ql/lib/**/*.qll"
-      - "*/ql/lib/ext/**/*.yml"
       - "misc/scripts/library-coverage/*.py"
       # input data files
       - "*/documentation/library-coverage/cwe-sink.csv"
@@ -31,11 +30,11 @@ jobs:
           GITHUB_CONTEXT: ${{ toJSON(github.event) }}
         run: echo "$GITHUB_CONTEXT"
       - name: Clone self (github/codeql) - MERGE
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: merge
       - name: Clone self (github/codeql) - BASE
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           fetch-depth: 2
           path: base

.github/workflows/csv-coverage-pr-comment.yml

@@ -20,7 +20,7 @@ jobs:
         GITHUB_CONTEXT: ${{ toJSON(github.event) }}
       run: echo "$GITHUB_CONTEXT"
     - name: Clone self (github/codeql)
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
     - name: Set up Python 3.8
       uses: actions/setup-python@v4
       with:

.github/workflows/csv-coverage-timeseries.yml

@@ -9,11 +9,11 @@ jobs:
 
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: script
       - name: Clone self (github/codeql) for analysis
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: codeqlModels
           fetch-depth: 0

.github/workflows/csv-coverage-update.yml

@@ -17,7 +17,7 @@ jobs:
           GITHUB_CONTEXT: ${{ toJSON(github.event) }}
         run: echo "$GITHUB_CONTEXT"
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: ql
           fetch-depth: 0

.github/workflows/csv-coverage.yml

@@ -13,11 +13,11 @@ jobs:
 
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: script
       - name: Clone self (github/codeql) for analysis
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: codeqlModels
           ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}

.github/workflows/fast-forward.yml

@@ -1,50 +0,0 @@
-# Fast-forwards the branch specified in BRANCH_NAME
-# to the github.ref/sha that this workflow is run on.
-# Used as part of the release process, to ensure
-# external query writers can always access a branch of github/codeql
-# that is compatible with the latest stable release.
-name: Fast-forward tracking branch for selected CodeQL version
-on:
-  workflow_dispatch:
-
-jobs:
-  fast-forward:
-    name: Fast-forward tracking branch for selected CodeQL version
-    runs-on: ubuntu-latest
-    if: github.repository == 'github/codeql'
-    permissions:
-      contents: write
-    env:
-      BRANCH_NAME: 'lgtm.com'
-    steps:
-      - name: Validate chosen branch
-        if: ${{ !startsWith(github.ref_name, 'codeql-cli-') }}
-        shell: bash
-        run: |
-          echo "::error ::The $BRANCH_NAME tracking branch should only be fast-forwarded to the tip of a codeql-cli-* branch, got $GITHUB_REF_NAME instead."
-          exit 1
-
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Git config
-        shell: bash
-        run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-
-      - name: Fetch
-        shell: bash
-        run: |
-          set -x
-          echo "Fetching $BRANCH_NAME"
-          # Explicitly unshallow and fetch to ensure the remote ref is available.
-          git fetch --unshallow origin "$BRANCH_NAME"
-          git checkout -b "$BRANCH_NAME" "origin/$BRANCH_NAME"
-
-      - name: Fast-forward
-        shell: bash
-        run: |
-          echo "Fast-forwarding $BRANCH_NAME to ${GITHUB_REF}@${GITHUB_SHA}"
-          git merge --ff-only "$GITHUB_SHA"
-          git push origin "$BRANCH_NAME"

.github/workflows/go-tests-other-os.yml

@@ -7,21 +7,19 @@ on:
       - .github/workflows/go-tests-other-os.yml
       - .github/actions/**
       - codeql-workspace.yml
-env:
-  GO_VERSION: '~1.21.0'
 jobs:
   test-mac:
     name: Test MacOS
     runs-on: macos-latest
     steps:
-      - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v4
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: 1.19
         id: go
 
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
       - name: Set up CodeQL CLI
         uses: ./.github/actions/fetch-codeql
@@ -49,14 +47,14 @@ jobs:
     name: Test Windows
     runs-on: windows-latest-xl
     steps:
-      - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v4
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: 1.19
         id: go
 
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
       - name: Set up CodeQL CLI
         uses: ./.github/actions/fetch-codeql

.github/workflows/go-tests.yml

@@ -15,21 +15,19 @@ on:
       - .github/workflows/go-tests.yml
       - .github/actions/**
       - codeql-workspace.yml
-env:
-  GO_VERSION: '~1.21.0'
 jobs:
   test-linux:
     name: Test Linux (Ubuntu)
     runs-on: ubuntu-latest-xl
     steps:
-      - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v4
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: 1.19
         id: go
 
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
       - name: Set up CodeQL CLI
         uses: ./.github/actions/fetch-codeql

.github/workflows/js-ml-tests.yml

@@ -0,0 +1,65 @@
+name: JS ML-powered queries tests
+
+on:
+  push:
+    paths:
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+      - .github/workflows/js-ml-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+      - "rc/*"
+  pull_request:
+    paths:
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+      - .github/workflows/js-ml-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+  workflow_dispatch:
+
+defaults:
+  run:
+    working-directory: javascript/ql/experimental/adaptivethreatmodeling
+
+jobs:
+  qltest:
+    name: Test QL
+    runs-on: ubuntu-latest-xl
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Install pack dependencies
+        run: |
+          for pack in modelbuilding src test; do
+            codeql pack install --mode verify -- "${pack}"
+          done
+      
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: js-ml-test
+
+      - name: Check QL compilation
+        run: |
+          codeql query compile \
+            --check-only \
+            --ram 50000 \
+            --additional-packs "${{ github.workspace }}" \
+            --threads=0 \
+            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
+            -- \
+            lib modelbuilding src
+
+      - name: Run QL tests
+        run: |
+          codeql test run \
+            --threads=0 \
+            --ram 50000 \
+            --additional-packs "${{ github.workspace }}" \
+            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
+            -- \
+            test

.github/workflows/mad_modelDiff.yml

@@ -27,12 +27,12 @@ jobs:
         slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
     steps:
       - name: Clone github/codeql from PR
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         if: github.event.pull_request
         with:
           path: codeql-pr
       - name: Clone github/codeql from main
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: codeql-main
           ref: main

.github/workflows/mad_regenerate-models.yml

@@ -27,11 +27,11 @@ jobs:
             ref: "placeholder"
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
       - name: Setup CodeQL binaries
         uses: ./.github/actions/fetch-codeql
       - name: Clone repositories
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           path: repos/${{ matrix.ref }}
           ref: ${{ matrix.ref }}

.github/workflows/qhelp-pr-preview.yml

@@ -43,7 +43,7 @@ jobs:
           if-no-files-found: error
           retention-days: 1
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 2
           persist-credentials: false

.github/workflows/ql-for-ql-build.yml

@@ -5,6 +5,13 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+    paths:
+      - "ql/**"
+      - "**.qll"
+      - "**.ql"
+      - "**.dbscheme"
+      - "**/qlpack.yml"
+      - ".github/workflows/ql-for-ql-build.yml"
 
 env:
   CARGO_TERM_COLOR: always
@@ -14,27 +21,67 @@ jobs:
     runs-on: ubuntu-latest-xl
     steps:
       ### Build the queries ###
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
+      - uses: actions/checkout@v3
+      - name: Find latest bundle
+        id: find-latest-bundle
+        uses: ./.github/actions/find-latest-bundle
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
         with:
           languages: javascript # does not matter
+          tools: ${{ steps.find-latest-bundle.outputs.url }}
+      - name: Get CodeQL version
+        id: get-codeql-version
+        run: |
+          echo "version=$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)" >> $GITHUB_OUTPUT
+        shell: bash
+        env:
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - uses: ./.github/actions/os-version
         id: os_version
+      - name: Cache entire pack
+        id: cache-pack
+        uses: actions/cache@v3
+        with:
+          path: ${{ runner.temp }}/pack
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-pack-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
+      - name: Cache queries
+        if: steps.cache-pack.outputs.cache-hit != 'true'
+        id: cache-queries
+        uses: actions/cache@v3
+        with:
+          path: ${{ runner.temp }}/queries
+          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
+      - name: Build query pack
+        if: steps.cache-queries.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: |
+          cd ql/ql/src
+          "${CODEQL}" pack create -j 16
+          mv .codeql/pack/codeql/ql/0.0.0 ${{ runner.temp }}/queries
+        env:
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+      - name: Move cache queries to pack
+        if: steps.cache-pack.outputs.cache-hit != 'true'
+        run: |
+          cp -r ${{ runner.temp }}/queries ${{ runner.temp }}/pack
+        env:
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+
       ### Build the extractor ###
       - name: Cache entire extractor
+        if: steps.cache-pack.outputs.cache-hit != 'true'
         id: cache-extractor
         uses: actions/cache@v3
         with:
           path: |
-            ql/extractor-pack/
-            ql/target/release/buramu
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ql/**/*.rs') }}
+            ql/target/release/ql-autobuilder
+            ql/target/release/ql-autobuilder.exe
+            ql/target/release/ql-extractor
+            ql/target/release/ql-extractor.exe
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
       - name: Cache cargo
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         uses: actions/cache@v3
         with:
           path: |
@@ -42,33 +89,77 @@ jobs:
             ~/.cargo/git
             ql/target
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
+      - name: Check formatting
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: cd ql; cargo fmt --all -- --check
+      - name: Build
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: cd ql; cargo build --verbose
+      - name: Run tests
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: cd ql; cargo test --verbose
       - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; ./scripts/create-extractor-pack.sh       
-        env:
-          GH_TOKEN: ${{ github.token }}   
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: run-ql-for-ql
-      - name: Make database and analyze
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: cd ql; cargo build --release
+      - name: Generate dbscheme
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
+
+      ### Package the queries and extractor ###
+      - name: Package pack
+        if: steps.cache-pack.outputs.cache-hit != 'true'
         run: |
-          ./ql/target/release/buramu | tee deprecated.blame # Add a blame file for the extractor to parse.
-          ${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB}
-          ${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls  --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-        env: 
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-          DB: ${{ runner.temp }}/DB
-          LGTM_INDEX_FILTERS: |
-            exclude:ql/ql/test
-            exclude:*/ql/lib/upgrades/
-            exclude:java/ql/integration-tests
-      - name: Upload sarif to code-scanning
-        uses: github/codeql-action/upload-sarif@v2
+          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats ${PACK}/
+          mkdir -p ${PACK}/tools/linux64
+          cp ql/target/release/ql-autobuilder  ${PACK}/tools/linux64/autobuilder
+          cp ql/target/release/ql-extractor ${PACK}/tools/linux64/extractor
+          chmod +x ${PACK}/tools/linux64/autobuilder
+          chmod +x ${PACK}/tools/linux64/extractor
+        env:
+          PACK: ${{ runner.temp }}/pack
+
+      ### Run the analysis ###
+      - name: Hack codeql-action options
+        run: |
+          JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .resolve.languages=["--search-path", $pack] | .database.init=["--search-path", $pack]')
+          echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
+        env:
+          PACK: ${{ runner.temp }}/pack
+
+      - name: Create CodeQL config file
+        run: |
+          echo "paths-ignore:" >> ${CONF}
+          echo "  - ql/ql/test" >> ${CONF}
+          echo "  - \"*/ql/lib/upgrades/\"" >> ${CONF}
+          echo "disable-default-queries: true" >> ${CONF}
+          echo "queries:" >> ${CONF}
+          echo "  - uses: ./ql/ql/src/codeql-suites/ql-code-scanning.qls" >> ${CONF}
+          echo "Config file: "
+          cat ${CONF}
+        env:
+          CONF: ./ql-for-ql-config.yml
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
         with:
-          sarif_file: ql-for-ql.sarif
-          category: ql-for-ql
+          languages: ql
+          db-location: ${{ runner.temp }}/db
+          config-file: ./ql-for-ql-config.yml
+          tools: ${{ steps.find-latest-bundle.outputs.url }}
+      - name: Move pack queries
+        run: |
+          cp -r ${PACK}/queries ql/ql/src
+        env:
+          PACK: ${{ runner.temp }}/pack
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@45955cb1830b640e2c1603ad72ad542a49d47b96
+        with:
+          category: "ql-for-ql"
+      - name: Copy sarif file to CWD
+        run: cp ../results/ql.sarif ./ql-for-ql.sarif
+      - name: Fixup the $scema in sarif  # Until https://github.com/microsoft/sarif-vscode-extension/pull/436/ is part in a stable release
+        run: |
+          sed -i 's/\$schema.*/\$schema": "https:\/\/raw.githubusercontent.com\/oasis-tcs\/sarif-spec\/master\/Schemata\/sarif-schema-2.1.0",/' ql-for-ql.sarif
       - name: Sarif as artifact
         uses: actions/upload-artifact@v3
         with:

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -21,11 +21,11 @@ jobs:
           - github/codeql
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version
@@ -42,7 +42,7 @@ jobs:
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           repository: ${{ matrix.repo }}
           path: ${{ github.workspace }}/repo
@@ -71,7 +71,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: measure
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
         with:
           name: measurements

.github/workflows/ql-for-ql-tests.yml

@@ -6,13 +6,11 @@ on:
     paths:
       - "ql/**"
       - codeql-workspace.yml
-      - .github/workflows/ql-for-ql-tests.yml
   pull_request:
     branches: [main]
     paths:
       - "ql/**"
       - codeql-workspace.yml
-      - .github/workflows/ql-for-ql-tests.yml
 
 env:
   CARGO_TERM_COLOR: always
@@ -21,10 +19,10 @@ jobs:
   qltest:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version
@@ -36,8 +34,6 @@ jobs:
             ~/.cargo/git
             ql/target
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/rust-toolchain.toml', 'ql/**/Cargo.lock') }}
-      - name: Check formatting
-        run: cd ql; cargo fmt --all -- --check
       - name: Build extractor
         run: |
           cd ql;
@@ -61,7 +57,7 @@ jobs:
     needs: [qltest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Install GNU tar 
         if: runner.os == 'macOS'
         run: |
@@ -69,7 +65,7 @@ jobs:
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version

.github/workflows/query-list.yml

@@ -20,7 +20,7 @@ jobs:
 
     steps:
     - name: Clone self (github/codeql)
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
       with:
         path: codeql 
     - name: Set up Python 3.8

.github/workflows/ruby-build.yml

@@ -42,15 +42,12 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Install GNU tar
         if: runner.os == 'macOS'
         run: |
           brew install gnu-tar
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
-      - name: Install cargo-cross
-        if: runner.os == 'Linux'
-        run: cargo install cross --version 0.2.5
       - uses: ./.github/actions/os-version
         id: os_version
       - name: Cache entire extractor
@@ -58,10 +55,12 @@ jobs:
         id: cache-extractor
         with:
           path: |
-            ruby/extractor/target/release/codeql-extractor-ruby
-            ruby/extractor/target/release/codeql-extractor-ruby.exe
-            ruby/extractor/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ruby/extractor/**/*.rs') }}
+            ruby/target/release/ruby-autobuilder
+            ruby/target/release/ruby-autobuilder.exe
+            ruby/target/release/ruby-extractor
+            ruby/target/release/ruby-extractor.exe
+            ruby/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}--${{ hashFiles('ruby/**/*.rs') }}
       - uses: actions/cache@v3
         if: steps.cache-extractor.outputs.cache-hit != 'true'
         with:
@@ -69,30 +68,22 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             ruby/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-rust-cargo-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/**/Cargo.lock') }}
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-rust-cargo-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}
       - name: Check formatting
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd extractor && cargo fmt --all -- --check
+        run: cargo fmt --all -- --check
       - name: Build
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd extractor && cargo build --verbose
+        run: cargo build --verbose
       - name: Run tests
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd extractor && cargo test --verbose
-      # On linux, build the extractor via cross in a centos7 container.
-      # This ensures we don't depend on glibc > 2.17.
-      - name: Release build (linux)
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os == 'Linux'
-        run: |
-          cd extractor
-          cross build --release
-          mv target/x86_64-unknown-linux-gnu/release/codeql-extractor-ruby target/release/
-      - name: Release build (windows and macos)
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os != 'Linux'
-        run: cd extractor && cargo build --release
+        run: cargo test --verbose
+      - name: Release build
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        run: cargo build --release
       - name: Generate dbscheme
         if: ${{ matrix.os == 'ubuntu-latest' && steps.cache-extractor.outputs.cache-hit != 'true'}}
-        run: extractor/target/release/codeql-extractor-ruby generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+        run: target/release/ruby-generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v3
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
@@ -107,13 +98,15 @@ jobs:
         with:
           name: extractor-${{ matrix.os }}
           path: |
-            ruby/extractor/target/release/codeql-extractor-ruby
-            ruby/extractor/target/release/codeql-extractor-ruby.exe
+            ruby/target/release/ruby-autobuilder
+            ruby/target/release/ruby-autobuilder.exe
+            ruby/target/release/ruby-extractor
+            ruby/target/release/ruby-extractor.exe
           retention-days: 1
   compile-queries:
     runs-on: ubuntu-latest-xl
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Fetch CodeQL
         uses: ./.github/actions/fetch-codeql
       - name: Cache compilation cache
@@ -123,29 +116,28 @@ jobs:
           key: ruby-build
       - name: Build Query Pack
         run: |
-          PACKS=${{ runner.temp }}/query-packs
-          rm -rf $PACKS
-          codeql pack create ../misc/suite-helpers --output "$PACKS"
-          codeql pack create ../shared/regex --output "$PACKS"
-          codeql pack create ../shared/ssa --output "$PACKS"
-          codeql pack create ../shared/tutorial --output "$PACKS"
-          codeql pack create ql/lib --output "$PACKS"
-          codeql pack create -j0 ql/src --output "$PACKS" --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-          PACK_FOLDER=$(readlink -f "$PACKS"/codeql/ruby-queries/*)
+          rm -rf target/packs
+          codeql pack create ../misc/suite-helpers --output target/packs
+          codeql pack create ../shared/regex --output target/packs
+          codeql pack create ../shared/ssa --output target/packs
+          codeql pack create ../shared/tutorial --output target/packs
+          codeql pack create ql/lib --output target/packs
+          codeql pack create -j0 ql/src --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
           codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
           (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
       - uses: actions/upload-artifact@v3
         with:
           name: codeql-ruby-queries
           path: |
-            ${{ runner.temp }}/query-packs/*
+            ruby/target/packs/*
           retention-days: 1
 
   package:
     runs-on: ubuntu-latest
     needs: [build, compile-queries]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
         with:
           name: ruby.dbscheme
@@ -166,10 +158,13 @@ jobs:
           mkdir -p ruby
           cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
           mkdir -p ruby/tools/{linux64,osx64,win64}
-          cp linux64/codeql-extractor-ruby ruby/tools/linux64/extractor
-          cp osx64/codeql-extractor-ruby ruby/tools/osx64/extractor
-          cp win64/codeql-extractor-ruby.exe ruby/tools/win64/extractor.exe
-          chmod +x ruby/tools/{linux64,osx64}/extractor
+          cp linux64/ruby-autobuilder ruby/tools/linux64/autobuilder
+          cp osx64/ruby-autobuilder ruby/tools/osx64/autobuilder
+          cp win64/ruby-autobuilder.exe ruby/tools/win64/autobuilder.exe
+          cp linux64/ruby-extractor ruby/tools/linux64/extractor
+          cp osx64/ruby-extractor ruby/tools/osx64/extractor
+          cp win64/ruby-extractor.exe ruby/tools/win64/extractor.exe
+          chmod +x ruby/tools/{linux64,osx64}/{autobuilder,extractor}
           zip -rq codeql-ruby.zip ruby
       - uses: actions/upload-artifact@v3
         with:
@@ -206,7 +201,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     needs: [package]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Fetch CodeQL
         uses: ./.github/actions/fetch-codeql
 
@@ -231,54 +226,3 @@ jobs:
         shell: bash
         run: |
           codeql database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls
-
-  # This is a copy of the 'test' job that runs in a centos7 container.
-  # This tests that the extractor works correctly on systems with an old glibc.
-  test-centos7:
-    defaults:
-      run:
-        working-directory: ${{ github.workspace }}
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-latest
-    container:
-      image: centos:centos7
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    needs: [package]
-    steps:
-      - name: Install gh cli
-        run: |
-          yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
-          # fetch-codeql requires unzip and jq
-          # jq is available in epel-release (https://docs.fedoraproject.org/en-US/epel/)
-          yum install -y gh unzip epel-release
-          yum install -y jq
-      - uses: actions/checkout@v3
-      - name: Fetch CodeQL
-        uses: ./.github/actions/fetch-codeql
-
-      # Due to a bug in Actions, we can't use runner.temp in the run blocks here.
-      # https://github.com/actions/runner/issues/2185
-
-      - name: Download Ruby bundle
-        uses: actions/download-artifact@v3
-        with:
-          name: codeql-ruby-bundle
-          path: ${{ runner.temp }}
-      - name: Unzip Ruby bundle
-        shell: bash
-        run: unzip -q -d "$RUNNER_TEMP"/ruby-bundle "$RUNNER_TEMP"/codeql-ruby-bundle.zip
-
-      - name: Run QL test
-        shell: bash
-        run: |
-          codeql test run --search-path "$RUNNER_TEMP"/ruby-bundle --additional-packs "$RUNNER_TEMP"/ruby-bundle ruby/ql/test/library-tests/ast/constants/
-      - name: Create database
-        shell: bash
-        run: |
-          codeql database create --search-path "$RUNNER_TEMP"/ruby-bundle --language ruby --source-root ruby/ql/test/library-tests/ast/constants/ ../database
-      - name: Analyze database
-        shell: bash
-        run: |
-          codeql database analyze --search-path "$RUNNER_TEMP"/ruby-bundle --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/ruby-dataset-measure.yml

@@ -27,14 +27,14 @@ jobs:
         repo: [rails/rails, discourse/discourse, spree/spree, ruby/ruby]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
 
       - uses: ./.github/actions/fetch-codeql
 
       - uses: ./ruby/actions/create-extractor-pack
 
       - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           repository: ${{ matrix.repo }}
           path: ${{ github.workspace }}/repo
@@ -59,7 +59,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: measure
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
         with:
           name: measurements

.github/workflows/ruby-qltest.yml

@@ -4,7 +4,6 @@ on:
   push:
     paths:
       - "ruby/**"
-      - "shared/**"
       - .github/workflows/ruby-build.yml
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
@@ -14,7 +13,6 @@ on:
   pull_request:
     paths:
       - "ruby/**"
-      - "shared/**"
       - .github/workflows/ruby-qltest.yml
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
@@ -33,7 +31,7 @@ jobs:
   qlupgrade:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - name: Check DB upgrade scripts
         run: |
@@ -54,7 +52,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - uses: ./ruby/actions/create-extractor-pack
       - name: Cache compilation cache

.github/workflows/swift.yml

@@ -5,7 +5,6 @@ on:
     paths:
       - "swift/**"
       - "misc/bazel/**"
-      - "misc/codegen/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**
@@ -16,12 +15,10 @@ on:
     branches:
       - main
       - rc/*
-      - codeql-cli-*
   push:
     paths:
       - "swift/**"
       - "misc/bazel/**"
-      - "misc/codegen/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**
@@ -31,7 +28,6 @@ on:
     branches:
       - main
       - rc/*
-      - codeql-cli-*
 
 jobs:
   # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks
@@ -39,31 +35,31 @@ jobs:
   build-and-test-macos:
     runs-on: macos-12-xl
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./swift/actions/build-and-test
   build-and-test-linux:
     runs-on: ubuntu-latest-xl
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./swift/actions/build-and-test
   qltests-linux:
     needs: build-and-test-linux
     runs-on: ubuntu-latest-xl
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./swift/actions/run-ql-tests
   qltests-macos:
     if : ${{ github.event_name == 'pull_request' }}
     needs: build-and-test-macos
     runs-on: macos-12-xl
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./swift/actions/run-ql-tests
   integration-tests-linux:
     needs: build-and-test-linux
     runs-on: ubuntu-latest-xl
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./swift/actions/run-integration-tests
   integration-tests-macos:
     if : ${{ github.event_name == 'pull_request' }}
@@ -71,13 +67,13 @@ jobs:
     runs-on: macos-12-xl
     timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./swift/actions/run-integration-tests
   codegen:
     if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: bazelbuild/setup-bazelisk@v2
       - uses: actions/setup-python@v4
         with:
@@ -102,6 +98,6 @@ jobs:
     if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - uses: ./swift/actions/database-upgrade-scripts

.github/workflows/sync-files.yml

@@ -14,9 +14,7 @@ jobs:
   sync:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Check synchronized files
         run: python config/sync-files.py
-      - name: Check dbscheme fragments
-        run: python config/sync-dbscheme-fragments.py
 

.github/workflows/tree-sitter-extractor-test.yml

@@ -1,46 +0,0 @@
-name: Test tree-sitter-extractor
-
-on:
-  push:
-    paths:
-      - "shared/tree-sitter-extractor/**"
-      - .github/workflows/tree-sitter-extractor-test.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "shared/tree-sitter-extractor/**"
-      - .github/workflows/tree-sitter-extractor-test.yml
-    branches:
-      - main
-      - "rc/*"
-
-env:
-  CARGO_TERM_COLOR: always
-
-defaults:
-  run:
-    working-directory: shared/tree-sitter-extractor
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Check formatting
-        run: cargo fmt --all -- --check
-      - name: Run tests
-        run: cargo test --verbose
-  fmt:
-    runs-on: ubuntu-latest  
-    steps:
-      - uses: actions/checkout@v4
-      - name: Check formatting
-        run: cargo fmt --check
-  clippy:
-    runs-on: ubuntu-latest  
-    steps:
-      - uses: actions/checkout@v4
-      - name: Run clippy
-        run: cargo clippy -- --no-deps -D warnings -A clippy::new_without_default -A clippy::too_many_arguments

.github/workflows/validate-change-notes.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql

.pre-commit-config.yaml

@@ -5,9 +5,9 @@ repos:
     rev: v3.2.0
     hooks:
       - id: trailing-whitespace
-        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)|.*\.patch
+        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
       - id: end-of-file-fixer
-        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)|.*\.patch
+        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: v13.0.1
@@ -19,12 +19,7 @@ repos:
     rev: v1.6.0
     hooks:
       - id: autopep8
-        files: ^misc/codegen/.*\.py
-
-  - repo: https://github.com/warchant/pre-commit-buildifier
-    rev: 0.0.2
-    hooks:
-      - id: buildifier
+        files: ^swift/.*\.py
 
   - repo: local
     hooks:
@@ -58,5 +53,5 @@ repos:
         name: Run Swift code generation unit tests
         files: ^swift/codegen/.*\.py$
         language: system
-        entry: bazel test //misc/codegen/test
+        entry: bazel test //swift/codegen/test
         pass_filenames: false

.vscode/tasks.json

@@ -22,22 +22,6 @@
                 "command": "${config:python.pythonPath}",
             },
             "problemMatcher": []
-        },
-        {
-            "label": "Accept .expected changes from CI",
-            "type": "process",
-            // Non-Windows OS will usually have Python 3 already installed at /usr/bin/python3.
-            "command": "python3",
-            "args": [
-                "misc/scripts/accept-expected-changes-from-ci.py"
-            ],
-            "group": "build",
-            "windows": {
-                // On Windows, use whatever Python interpreter is configured for this workspace. The default is
-                // just `python`, so if Python is already on the path, this will find it.
-                "command": "${config:python.pythonPath}",
-            },
-            "problemMatcher": []
         }
     ]
-}
+}

CODEOWNERS

@@ -6,8 +6,8 @@
 /python/ @github/codeql-python
 /ruby/ @github/codeql-ruby
 /swift/ @github/codeql-swift
-/misc/codegen/ @github/codeql-swift
 /java/kotlin-extractor/ @github/codeql-kotlin
+/java/kotlin-explorer/ @github/codeql-kotlin
 
 # ML-powered queries
 /javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
@@ -39,6 +39,3 @@ WORKSPACE.bazel @github/codeql-ci-reviewers
 /.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
 /.github/workflows/ruby-* @github/codeql-ruby
 /.github/workflows/swift.yml @github/codeql-swift
-
-# Misc
-/misc/scripts/accept-expected-changes-from-ci.py @RasmusWL

CONTRIBUTING.md

@@ -14,16 +14,14 @@ If you have an idea for a query that you would like to share with other CodeQL u
 
 1. **Directory structure**
 
-    There are eight language-specific query directories in this repository:
+    There are six language-specific query directories in this repository:
 
       * C/C++: `cpp/ql/src`
       * C#: `csharp/ql/src`
-      * Go: `go/ql/src`
-      * Java/Kotlin: `java/ql/src`
+      * Java: `java/ql/src`
       * JavaScript: `javascript/ql/src`
       * Python: `python/ql/src`
       * Ruby: `ruby/ql/src`
-      * Swift: `swift/ql/src`
 
     Each language-specific directory contains further subdirectories that group queries based on their `@tags` or purpose.
     - Experimental queries and libraries are stored in the `experimental` subdirectory within each language-specific directory in the [CodeQL repository](https://github.com/github/codeql). For example, experimental Java queries and libraries are stored in `java/ql/src/experimental` and any corresponding tests in `java/ql/test/experimental`.

README.md

@@ -10,8 +10,6 @@ There is [extensive documentation](https://codeql.github.com/docs/) on getting s
 
 We welcome contributions to our standard library and standard checks. Do you have an idea for a new check, or how to improve an existing query? Then please go ahead and open a pull request! Before you do, though, please take the time to read our [contributing guidelines](CONTRIBUTING.md). You can also consult our [style guides](https://github.com/github/codeql/tree/main/docs) to learn how to format your code for consistency and clarity, how to write query metadata, and how to write query help documentation for your query.
 
-For information on contributing to CodeQL documentation, see the "[contributing guide](docs/codeql/CONTRIBUTING.md)" for docs.
-
 ## License
 
 The code in this repository is licensed under the [MIT License](LICENSE) by [GitHub](https://github.com).

codeql-workspace.yml

@@ -4,8 +4,6 @@ provide:
   - "*/ql/test/qlpack.yml"
   - "*/ql/examples/qlpack.yml"
   - "*/ql/consistency-queries/qlpack.yml"
-  - "*/ql/automodel/src/qlpack.yml"
-  - "*/ql/automodel/test/qlpack.yml"
   - "shared/*/qlpack.yml"
   - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
   - "go/ql/config/legacy-support/qlpack.yml"

config/dbscheme-fragments.json

@@ -1,33 +0,0 @@
-{
-  "files": [
-    "javascript/ql/lib/semmlecode.javascript.dbscheme",
-    "python/ql/lib/semmlecode.python.dbscheme",
-    "ruby/ql/lib/ruby.dbscheme",
-    "ql/ql/src/ql.dbscheme"
-  ],
-  "fragments": [
-    "/*- External data -*/",
-    "/*- Files and folders -*/",
-    "/*- Diagnostic messages -*/",
-    "/*- Diagnostic messages: severity -*/",
-    "/*- Source location prefix -*/",
-    "/*- Lines of code -*/",
-    "/*- Configuration files with key value pairs -*/",
-    "/*- YAML -*/",
-    "/*- XML Files -*/",
-    "/*- XML: sourceline -*/",
-    "/*- DEPRECATED: External defects and metrics -*/",
-    "/*- DEPRECATED: Snapshot date -*/",
-    "/*- DEPRECATED: Duplicate code -*/",
-    "/*- DEPRECATED: Version control data -*/",
-    "/*- JavaScript-specific part -*/",
-    "/*- Ruby dbscheme -*/",
-    "/*- Erb dbscheme -*/",
-    "/*- QL dbscheme -*/",
-    "/*- Dbscheme dbscheme -*/",
-    "/*- Yaml dbscheme -*/",
-    "/*- Blame dbscheme -*/",
-    "/*- JSON dbscheme -*/",
-    "/*- Python dbscheme -*/"
-  ]
-}

config/identical-files.json

@@ -1,41 +1,65 @@
 {
-  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift Legacy Configuration": [
-    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl1.qll",
+  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift": [
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl2.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl3.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl4.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl5.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl6.qll",
-    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl1.qll",
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForSerializability.qll",
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplForOnActivityResult.qll",
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl4.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplLocal.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl1.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl1.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl2.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl1.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplForContentDataFlow.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll",
     "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl2.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl1.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl1.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl1.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForRegExp.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForHttpClientLibraries.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForPathname.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"
   ],
-  "TaintTracking Legacy Configuration Java/C++/C#/Go/Python/Ruby/Swift": [
+  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift Common": [
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll",
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImplCommon.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll"
+  ],
+  "TaintTracking::Configuration Java/C++/C#/Go/Python/Ruby/Swift": [
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
@@ -53,6 +77,16 @@
     "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
   ],
+  "DataFlow Java/C++/C#/Python/Ruby/Swift Consistency checks": [
+    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll",
+    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplConsistency.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplConsistency.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplConsistency.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplConsistency.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplConsistency.qll"
+  ],
   "DataFlow Java/C#/Go/Ruby/Python/Swift Flow Summaries": [
     "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll",
@@ -69,10 +103,6 @@
     "java/ql/src/utils/modelgenerator/internal/CaptureModels.qll",
     "csharp/ql/src/utils/modelgenerator/internal/CaptureModels.qll"
   ],
-  "Model as Data Generation Java/C# - CaptureModelsPrinting": [
-    "java/ql/src/utils/modelgenerator/internal/CaptureModelsPrinting.qll",
-    "csharp/ql/src/utils/modelgenerator/internal/CaptureModelsPrinting.qll"
-  ],
   "Sign Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/Sign.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/Sign.qll"
@@ -229,11 +259,6 @@
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/IRBlockImports.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/IRBlockImports.qll"
   ],
-  "C++ IR IRConsistencyImports": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/IRConsistencyImports.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/IRConsistencyImports.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/IRConsistencyImports.qll"
-  ],
   "C++ IR IRFunctionImports": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/IRFunctionImports.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/IRFunctionImports.qll",
@@ -459,17 +484,17 @@
   "SensitiveDataHeuristics Python/JS": [
     "javascript/ql/lib/semmle/javascript/security/internal/SensitiveDataHeuristics.qll",
     "python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll",
-    "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll",
-    "swift/ql/lib/codeql/swift/security/internal/SensitiveDataHeuristics.qll"
+    "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll"
+  ],
+  "CFG": [
+    "csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll",
+    "ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll",
+    "swift/ql/lib/codeql/swift/controlflow/internal/ControlFlowGraphImplShared.qll"
   ],
   "TypeTracker": [
     "python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll",
     "ruby/ql/lib/codeql/ruby/typetracking/TypeTracker.qll"
   ],
-  "SummaryTypeTracker": [
-    "python/ql/lib/semmle/python/dataflow/new/internal/SummaryTypeTracker.qll",
-    "ruby/ql/lib/codeql/ruby/typetracking/internal/SummaryTypeTracker.qll"
-  ],
   "AccessPathSyntax": [
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/AccessPathSyntax.qll",
     "go/ql/lib/semmle/go/dataflow/internal/AccessPathSyntax.qll",
@@ -545,9 +570,5 @@
   "EncryptionKeySizes Python/Java": [
     "python/ql/lib/semmle/python/security/internal/EncryptionKeySizes.qll",
     "java/ql/lib/semmle/code/java/security/internal/EncryptionKeySizes.qll"
-  ],
-  "Python model summaries test extension": [
-    "python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ext.yml",
-    "python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ext.yml"
   ]
 }

config/sync-dbscheme-fragments.py

@@ -1,86 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import json
-import os
-import pathlib
-import re
-
-
-def make_groups(blocks):
-    groups = {}
-    for block in blocks:
-        groups.setdefault("".join(block["lines"]), []).append(block)
-    return list(groups.values())
-
-
-def validate_fragments(fragments):
-    ok = True
-    for header, blocks in fragments.items():
-        groups = make_groups(blocks)
-        if len(groups) > 1:
-            ok = False
-            print("Warning: dbscheme fragments with header '{}' are different for {}".format(header, ["{}:{}:{}".format(
-                group[0]["file"], group[0]["start"], group[0]["end"]) for group in groups]))
-    return ok
-
-
-def main():
-    script_path = os.path.realpath(__file__)
-    script_dir = os.path.dirname(script_path)
-    parser = argparse.ArgumentParser(
-        prog=os.path.basename(script_path),
-        description='Sync dbscheme fragments across files.'
-    )
-    parser.add_argument('files', metavar='dbscheme_file', type=pathlib.Path, nargs='*', default=[],
-                        help='dbscheme files to check')
-    args = parser.parse_args()
-
-    with open(os.path.join(script_dir, "dbscheme-fragments.json"), "r") as f:
-        config = json.load(f)
-
-    fragment_headers = set(config["fragments"])
-    fragments = {}
-    ok = True
-    for file in args.files + config["files"]:
-        with open(os.path.join(os.path.dirname(script_dir), file), "r") as dbscheme:
-            header = None
-            line_number = 1
-            block = {"file": file, "start": line_number,
-                     "end": None, "lines": []}
-
-            def end_block():
-                block["end"] = line_number - 1
-                if len(block["lines"]) > 0:
-                    if header is None:
-                        if re.match(r'(?m)\A(\s|//.*$|/\*(\**[^\*])*\*+/)*\Z', "".join(block["lines"])):
-                            # Ignore comments at the beginning of the file
-                            pass
-                        else:
-                            ok = False
-                            print("Warning: dbscheme fragment without header: {}:{}:{}".format(
-                                block["file"], block["start"], block["end"]))
-                    else:
-                        fragments.setdefault(header, []).append(block)
-            for line in dbscheme:
-                m = re.match(r"^\/\*-.*-\*\/$", line)
-                if m:
-                    end_block()
-                    header = line.strip()
-                    if header not in fragment_headers:
-                        ok = False
-                        print("Warning: unknown header for dbscheme fragment: '{}': {}:{}".format(
-                            header, file, line_number))
-                    block = {"file": file, "start": line_number,
-                             "end": None, "lines": []}
-                block["lines"].append(line)
-                line_number += 1
-            block["lines"].append('\n')
-            line_number += 1
-            end_block()
-    if not ok or not validate_fragments(fragments):
-        exit(1)
-
-
-if __name__ == "__main__":
-    main()

cpp/BUILD.bazel

@@ -1,17 +1,12 @@
-load("@rules_pkg//:mappings.bzl", "pkg_filegroup")
-
 package(default_visibility = ["//visibility:public"])
 
+load("@rules_pkg//:mappings.bzl", "pkg_filegroup")
+
 alias(
     name = "dbscheme",
     actual = "//cpp/ql/lib:dbscheme",
 )
 
-alias(
-    name = "dbscheme-stats",
-    actual = "//cpp/ql/lib:dbscheme-stats",
-)
-
 pkg_filegroup(
     name = "db-files",
     srcs = [

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -1,6 +1,5 @@
 using Xunit;
 using Semmle.Autobuild.Shared;
-using Semmle.Util;
 using System.Collections.Generic;
 using System;
 using System.Linq;
@@ -76,15 +75,6 @@ namespace Semmle.Autobuild.Cpp.Tests
             throw new ArgumentException("Missing RunProcess " + pattern);
         }
 
-        int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
-        {
-            var ret = (this as IBuildActions).RunProcess(cmd, args, workingDirectory, env, out var stdout);
-
-            stdout.ForEach(line => onOutput(line));
-
-            return ret;
-        }
-
         public IList<string> DirectoryDeleteIn = new List<string>();
 
         void IBuildActions.DirectoryDelete(string dir, bool recursive)
@@ -141,14 +131,6 @@ namespace Semmle.Autobuild.Cpp.Tests
 
         bool IBuildActions.IsWindows() => IsWindows;
 
-        public bool IsMacOs { get; set; }
-
-        bool IBuildActions.IsMacOs() => IsMacOs;
-
-        public bool IsArm { get; set; }
-
-        bool IBuildActions.IsArm() => IsArm;
-
         string IBuildActions.PathCombine(params string[] parts)
         {
             return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));
@@ -194,15 +176,6 @@ namespace Semmle.Autobuild.Cpp.Tests
             if (!DownloadFiles.Contains((address, fileName)))
                 throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");
         }
-
-        public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new TestDiagnosticWriter();
-    }
-
-    internal class TestDiagnosticWriter : IDiagnosticsWriter
-    {
-        public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
-
-        public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
     }
 
     /// <summary>
@@ -262,7 +235,6 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
-            Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
             Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
@@ -327,7 +299,7 @@ namespace Semmle.Autobuild.Cpp.Tests
         {
             Actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test.sln -DisableParallelProcessing"] = 1;
             Actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test.sln -DisableParallelProcessing"] = 0;
-            Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program^ Files^ ^(x86^)\Microsoft^ Visual^ Studio^ 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && msbuild C:\Project\test.sln /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"""] = 0;
+            Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && msbuild C:\Project\test.sln /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"""] = 0;
             Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0;

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
     <Nullable>enable</Nullable>

cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs

@@ -1,5 +1,4 @@
 using Semmle.Autobuild.Shared;
-using Semmle.Util;
 
 namespace Semmle.Autobuild.Cpp
 {
@@ -22,7 +21,7 @@ namespace Semmle.Autobuild.Cpp
 
     public class CppAutobuilder : Autobuilder<CppAutobuildOptions>
     {
-        public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options, new DiagnosticClassifier()) { }
+        public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options) { }
 
         public override BuildScript GetBuildScript()
         {

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <AssemblyName>Semmle.Autobuild.Cpp</AssemblyName>
     <RootNamespace>Semmle.Autobuild.Cpp</RootNamespace>
     <ApplicationIcon />

cpp/downgrades/19887dbd33327fb07d54251786e0cb2578539775/upgrade.properties

@@ -1,4 +0,0 @@
-description: Revert support for repeated initializers, which are allowed in C with designated initializers.
-compatibility: full
-aggregate_field_init.rel: reorder aggregate_field_init.rel (int aggregate, int initializer, int field, int position) aggregate initializer field
-aggregate_array_init.rel: reorder aggregate_array_init.rel (int aggregate, int initializer, int element_index, int position) aggregate initializer element_index

cpp/downgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/upgrade.properties

@@ -1,2 +0,0 @@
-description: Remove _Float128 type
-compatibility: full

cpp/downgrades/dbe9c8eb5fc6f54b7ae08c7317d0795b24961564/upgrade.properties

@@ -1,2 +0,0 @@
-description: Make __is_trivial a builtin operation
-compatibility: full

cpp/downgrades/qlpack.yml

@@ -2,4 +2,3 @@ name: codeql/cpp-downgrades
 groups: cpp
 downgrades: .
 library: true
-warnOnImplicitThis: true

cpp/ql/examples/qlpack.yml

@@ -4,4 +4,3 @@ groups:
   - examples
 dependencies:
   codeql/cpp-all: ${workspace}
-warnOnImplicitThis: true

cpp/ql/examples/queries.xml

@@ -0,0 +1 @@
+<queries language="cpp"/>

cpp/ql/lib/BUILD.bazel

@@ -1,7 +1,7 @@
-load("@rules_pkg//:mappings.bzl", "pkg_files")
-
 package(default_visibility = ["//cpp:__pkg__"])
 
+load("@rules_pkg//:mappings.bzl", "pkg_files")
+
 pkg_files(
     name = "dbscheme",
     srcs = ["semmlecode.cpp.dbscheme"],

cpp/ql/lib/CHANGELOG.md

@@ -1,192 +1,3 @@
-## 0.9.3
-
-No user-facing changes.
-
-## 0.9.2
-
-### Deprecated APIs
-
-* `getAllocatorCall` on `DeleteExpr` and `DeleteArrayExpr` has been deprecated. `getDeallocatorCall` should be used instead.
-
-### New Features
-
-* Added `DeleteOrDeleteArrayExpr` as a super type of `DeleteExpr` and `DeleteArrayExpr`
-
-### Minor Analysis Improvements
-
-* `delete` and `delete[]` are now modeled as calls to the relevant `operator delete` in the IR. In the case of a dynamic delete call a new instruction `VirtualDeleteFunctionAddress` is used to represent a function that dispatches to the correct delete implementation.
-* Only the 2 level indirection of `argv` (corresponding to `**argv`) is consided for `FlowSource`.
-
-## 0.9.1
-
-No user-facing changes.
-
-## 0.9.0
-
-### Breaking Changes
-
-* The `shouldPrintFunction` predicate from `PrintAstConfiguration` has been replaced by `shouldPrintDeclaration`. Users should now override `shouldPrintDeclaration` if they want to limit the declarations that should be printed.
-* The `shouldPrintFunction` predicate from `PrintIRConfiguration` has been replaced by `shouldPrintDeclaration`. Users should now override `shouldPrintDeclaration` if they want to limit the declarations that should be printed.
-
-### Major Analysis Improvements
-
-* The `PrintAST` library now also prints global and namespace variables and their initializers.
-
-### Minor Analysis Improvements
-
-* The `_Float128x` type is no longer exposed as a builtin type. As this type could not occur any code base, this should only affect queries that explicitly looked at the builtin types.
-
-## 0.8.1
-
-### Deprecated APIs
-
-* The library `semmle.code.cpp.dataflow.DataFlow` has been deprecated. Please use `semmle.code.cpp.dataflow.new.DataFlow` instead.
-
-### New Features
-
-* The `DataFlow::StateConfigSig` signature module has gained default implementations for `isBarrier/2` and `isAdditionalFlowStep/4`. 
-  Hence it is no longer needed to provide `none()` implementations of these predicates if they are not needed.
-
-### Minor Analysis Improvements
-
-* Data flow configurations can now include a predicate `neverSkip(Node node)`
-  in order to ensure inclusion of certain nodes in the path explanations. The
-  predicate defaults to the end-points of the additional flow steps provided in
-  the configuration, which means that such steps now always are visible by
-  default in path explanations.
-* The `IRGuards` library has improved handling of pointer addition and subtraction operations.
-
-## 0.8.0
-
-### New Features
-
-* The `ProductFlow::StateConfigSig` signature now includes default predicates for `isBarrier1`, `isBarrier2`, `isAdditionalFlowStep1`, and `isAdditionalFlowStep1`. Hence, it is no longer needed to provide `none()` implementations of these predicates if they are not needed.
-
-### Minor Analysis Improvements
-
-* Deleted the deprecated `getURL` predicate from the `Container`, `Folder`, and `File` classes. Use the `getLocation` predicate instead.
-
-## 0.7.4
-
-No user-facing changes.
-
-## 0.7.3
-
-### Minor Analysis Improvements
-
-* Deleted the deprecated `hasCopyConstructor` predicate from the `Class` class in `Class.qll`.
-* Deleted many deprecated predicates and classes with uppercase `AST`, `SSA`, `CFG`, `API`, etc. in their names. Use the PascalCased versions instead.
-* Deleted the deprecated `CodeDuplication.qll` file.
-
-## 0.7.2
-
-### New Features
-
-* Added an AST-based interface (`semmle.code.cpp.rangeanalysis.new.RangeAnalysis`) for the relative range analysis library.
-* A new predicate `BarrierGuard::getAnIndirectBarrierNode` has been added to the new dataflow library (`semmle.code.cpp.dataflow.new.DataFlow`) to mark indirect expressions as barrier nodes using the `BarrierGuard` API.
-
-### Major Analysis Improvements
-
-* In the intermediate representation, handling of control flow after non-returning calls has been improved. This should remove false positives in queries that use the intermedite representation or libraries based on it, including the new data flow library.
-
-### Minor Analysis Improvements
-
-* The `StdNamespace` class now also includes all inline namespaces that are children of `std` namespace.
-* The new dataflow (`semmle.code.cpp.dataflow.new.DataFlow`) and taint-tracking libraries (`semmle.code.cpp.dataflow.new.TaintTracking`) now support tracking flow through static local variables.
-
-## 0.7.1
-
-No user-facing changes.
-
-## 0.7.0
-
-### Breaking Changes
-
-* The internal `SsaConsistency` module has been moved from `SSAConstruction` to `SSAConsitency`, and the deprecated `SSAConsistency` module has been removed.
-
-### Deprecated APIs
-
-* The single-parameter predicates `ArrayOrVectorAggregateLiteral.getElementExpr` and `ClassAggregateLiteral.getFieldExpr` have been deprecated in favor of `ArrayOrVectorAggregateLiteral.getAnElementExpr` and `ClassAggregateLiteral.getAFieldExpr`.
-* The recently introduced new data flow and taint tracking APIs have had a
-  number of module and predicate renamings. The old APIs remain in place for
-  now.
-* The `SslContextCallAbstractConfig`, `SslContextCallConfig`, `SslContextCallBannedProtocolConfig`, `SslContextCallTls12ProtocolConfig`, `SslContextCallTls13ProtocolConfig`, `SslContextCallTlsProtocolConfig`, `SslContextFlowsToSetOptionConfig`, `SslOptionConfig` dataflow configurations from `BoostorgAsio` have been deprecated. Please use `SslContextCallConfigSig`, `SslContextCallGlobal`, `SslContextCallFlow`, `SslContextCallBannedProtocolFlow`, `SslContextCallTls12ProtocolFlow`, `SslContextCallTls13ProtocolFlow`, `SslContextCallTlsProtocolFlow`, `SslContextFlowsToSetOptionFlow`.
-
-### New Features
-
-* Added overridable predicates `getSizeExpr` and `getSizeMult` to the `BufferAccess` class (`semmle.code.cpp.security.BufferAccess.qll`). This makes it possible to model a larger class of buffer reads and writes using the library.
-
-### Minor Analysis Improvements
-
-* The `BufferAccess` library (`semmle.code.cpp.security.BufferAccess`) no longer matches buffer accesses inside unevaluated contexts (such as inside `sizeof` or `decltype` expressions). As a result, queries using this library may see fewer false positives.
-
-### Bug Fixes
-
-* Fixed some accidental predicate visibility in the backwards-compatible wrapper for data flow configurations. In particular `DataFlow::hasFlowPath`, `DataFlow::hasFlow`, `DataFlow::hasFlowTo`, and `DataFlow::hasFlowToExpr` were accidentally exposed in a single version.
-
-## 0.6.1
-
-No user-facing changes.
-
-## 0.6.0
-
-### Breaking Changes
-
-* The `semmle.code.cpp.commons.Buffer` and `semmle.code.cpp.commons.NullTermination` libraries no longer expose `semmle.code.cpp.dataflow.DataFlow`. Please import `semmle.code.cpp.dataflow.DataFlow` directly.
-
-### Deprecated APIs
-
-* The `WriteConfig` taint tracking configuration has been deprecated. Please use `WriteFlow`.
-
-### New Features
-
-* Added support for merging two `PathGraph`s via disjoint union to allow results from multiple data flow computations in a single `path-problem` query.
-
-### Major Analysis Improvements
-
-* A new C/C++ dataflow library (`semmle.code.cpp.dataflow.new.DataFlow`) has been added.
-  The new library behaves much more like the dataflow library of other CodeQL supported
-  languages by following use-use dataflow paths instead of def-use dataflow paths.
-  The new library also better supports dataflow through indirections, and new predicates
-  such as `Node::asIndirectExpr` have been added to facilitate working with indirections.
-
-  The `semmle.code.cpp.ir.dataflow.DataFlow` library is now identical to the new
-  `semmle.code.cpp.dataflow.new.DataFlow` library.
-* The main data flow and taint tracking APIs have been changed. The old APIs
-  remain in place for now and translate to the new through a
-  backwards-compatible wrapper. If multiple configurations are in scope
-  simultaneously, then this may affect results slightly. The new API is quite
-  similar to the old, but makes use of a configuration module instead of a
-  configuration class.
-
-### Minor Analysis Improvements
-
-* Deleted the deprecated `hasGeneratedCopyConstructor` and `hasGeneratedCopyAssignmentOperator` predicates from the `Folder` class.
-* Deleted the deprecated `getPath` and `getFolder` predicates from the `XmlFile` class.
-* Deleted the deprecated `getMustlockFunction`, `getTrylockFunction`, `getLockFunction`, and `getUnlockFunction` predicates from the `MutexType` class.
-* Deleted the deprecated `getPosInBasicBlock` predicate from the `SubBasicBlock` class.
-* Deleted the deprecated `getExpr` predicate from the `PointerDereferenceExpr` class.
-* Deleted the deprecated `getUseInstruction` and `getDefinitionInstruction` predicates from the `Operand` class.
-* Deleted the deprecated `isInParameter`, `isInParameterPointer`, and `isInQualifier` predicates from the `FunctionInput` class.
-* Deleted the deprecated `isOutParameterPointer`, `isOutQualifier`, `isOutReturnValue`, and `isOutReturnPointer` predicate from the `FunctionOutput` class.
-* Deleted the deprecated 3-argument `isGuardPhi` predicate from the `RangeSsaDefinition` class.
-
-## 0.5.4
-
-No user-facing changes.
-
-## 0.5.3
-
-No user-facing changes.
-
-## 0.5.2
-
-No user-facing changes.
-
-## 0.5.1
-
-No user-facing changes.
-
 ## 0.5.0
 
 ### Breaking Changes

cpp/ql/lib/change-notes/2023-09-06-as-defining-argument-off-by-one-fix.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The `DataFlow::asDefiningArgument` predicate now takes its argument from the range starting at `1` instead of `2`. Queries that depend on the single-parameter version of `DataFlow::asDefiningArgument` should have their arguments updated accordingly.

cpp/ql/lib/change-notes/2023-09-07-return-from-end.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* Treat functions that reach the end of the function as returning in the IR.
-  They used to be treated as unreachable but it is allowed in C. 

cpp/ql/lib/change-notes/2023-09-08-more-unreachble.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* Functions that do not return due to calling functions that don't return (e.g. `exit`) are now detected as
- non-returning in the IR and dataflow.

cpp/ql/lib/change-notes/released/0.5.1.md

@@ -1,3 +0,0 @@
-## 0.5.1
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.5.2.md

@@ -1,3 +0,0 @@
-## 0.5.2
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.5.3.md

@@ -1,3 +0,0 @@
-## 0.5.3
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.5.4.md

@@ -1,3 +0,0 @@
-## 0.5.4
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.6.0.md

@@ -1,42 +0,0 @@
-## 0.6.0
-
-### Breaking Changes
-
-* The `semmle.code.cpp.commons.Buffer` and `semmle.code.cpp.commons.NullTermination` libraries no longer expose `semmle.code.cpp.dataflow.DataFlow`. Please import `semmle.code.cpp.dataflow.DataFlow` directly.
-
-### Deprecated APIs
-
-* The `WriteConfig` taint tracking configuration has been deprecated. Please use `WriteFlow`.
-
-### New Features
-
-* Added support for merging two `PathGraph`s via disjoint union to allow results from multiple data flow computations in a single `path-problem` query.
-
-### Major Analysis Improvements
-
-* A new C/C++ dataflow library (`semmle.code.cpp.dataflow.new.DataFlow`) has been added.
-  The new library behaves much more like the dataflow library of other CodeQL supported
-  languages by following use-use dataflow paths instead of def-use dataflow paths.
-  The new library also better supports dataflow through indirections, and new predicates
-  such as `Node::asIndirectExpr` have been added to facilitate working with indirections.
-
-  The `semmle.code.cpp.ir.dataflow.DataFlow` library is now identical to the new
-  `semmle.code.cpp.dataflow.new.DataFlow` library.
-* The main data flow and taint tracking APIs have been changed. The old APIs
-  remain in place for now and translate to the new through a
-  backwards-compatible wrapper. If multiple configurations are in scope
-  simultaneously, then this may affect results slightly. The new API is quite
-  similar to the old, but makes use of a configuration module instead of a
-  configuration class.
-
-### Minor Analysis Improvements
-
-* Deleted the deprecated `hasGeneratedCopyConstructor` and `hasGeneratedCopyAssignmentOperator` predicates from the `Folder` class.
-* Deleted the deprecated `getPath` and `getFolder` predicates from the `XmlFile` class.
-* Deleted the deprecated `getMustlockFunction`, `getTrylockFunction`, `getLockFunction`, and `getUnlockFunction` predicates from the `MutexType` class.
-* Deleted the deprecated `getPosInBasicBlock` predicate from the `SubBasicBlock` class.
-* Deleted the deprecated `getExpr` predicate from the `PointerDereferenceExpr` class.
-* Deleted the deprecated `getUseInstruction` and `getDefinitionInstruction` predicates from the `Operand` class.
-* Deleted the deprecated `isInParameter`, `isInParameterPointer`, and `isInQualifier` predicates from the `FunctionInput` class.
-* Deleted the deprecated `isOutParameterPointer`, `isOutQualifier`, `isOutReturnValue`, and `isOutReturnPointer` predicate from the `FunctionOutput` class.
-* Deleted the deprecated 3-argument `isGuardPhi` predicate from the `RangeSsaDefinition` class.

cpp/ql/lib/change-notes/released/0.6.1.md

@@ -1,3 +0,0 @@
-## 0.6.1
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.7.0.md

@@ -1,25 +0,0 @@
-## 0.7.0
-
-### Breaking Changes
-
-* The internal `SsaConsistency` module has been moved from `SSAConstruction` to `SSAConsitency`, and the deprecated `SSAConsistency` module has been removed.
-
-### Deprecated APIs
-
-* The single-parameter predicates `ArrayOrVectorAggregateLiteral.getElementExpr` and `ClassAggregateLiteral.getFieldExpr` have been deprecated in favor of `ArrayOrVectorAggregateLiteral.getAnElementExpr` and `ClassAggregateLiteral.getAFieldExpr`.
-* The recently introduced new data flow and taint tracking APIs have had a
-  number of module and predicate renamings. The old APIs remain in place for
-  now.
-* The `SslContextCallAbstractConfig`, `SslContextCallConfig`, `SslContextCallBannedProtocolConfig`, `SslContextCallTls12ProtocolConfig`, `SslContextCallTls13ProtocolConfig`, `SslContextCallTlsProtocolConfig`, `SslContextFlowsToSetOptionConfig`, `SslOptionConfig` dataflow configurations from `BoostorgAsio` have been deprecated. Please use `SslContextCallConfigSig`, `SslContextCallGlobal`, `SslContextCallFlow`, `SslContextCallBannedProtocolFlow`, `SslContextCallTls12ProtocolFlow`, `SslContextCallTls13ProtocolFlow`, `SslContextCallTlsProtocolFlow`, `SslContextFlowsToSetOptionFlow`.
-
-### New Features
-
-* Added overridable predicates `getSizeExpr` and `getSizeMult` to the `BufferAccess` class (`semmle.code.cpp.security.BufferAccess.qll`). This makes it possible to model a larger class of buffer reads and writes using the library.
-
-### Minor Analysis Improvements
-
-* The `BufferAccess` library (`semmle.code.cpp.security.BufferAccess`) no longer matches buffer accesses inside unevaluated contexts (such as inside `sizeof` or `decltype` expressions). As a result, queries using this library may see fewer false positives.
-
-### Bug Fixes
-
-* Fixed some accidental predicate visibility in the backwards-compatible wrapper for data flow configurations. In particular `DataFlow::hasFlowPath`, `DataFlow::hasFlow`, `DataFlow::hasFlowTo`, and `DataFlow::hasFlowToExpr` were accidentally exposed in a single version.

cpp/ql/lib/change-notes/released/0.7.1.md

@@ -1,3 +0,0 @@
-## 0.7.1
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.7.2.md

@@ -1,15 +0,0 @@
-## 0.7.2
-
-### New Features
-
-* Added an AST-based interface (`semmle.code.cpp.rangeanalysis.new.RangeAnalysis`) for the relative range analysis library.
-* A new predicate `BarrierGuard::getAnIndirectBarrierNode` has been added to the new dataflow library (`semmle.code.cpp.dataflow.new.DataFlow`) to mark indirect expressions as barrier nodes using the `BarrierGuard` API.
-
-### Major Analysis Improvements
-
-* In the intermediate representation, handling of control flow after non-returning calls has been improved. This should remove false positives in queries that use the intermedite representation or libraries based on it, including the new data flow library.
-
-### Minor Analysis Improvements
-
-* The `StdNamespace` class now also includes all inline namespaces that are children of `std` namespace.
-* The new dataflow (`semmle.code.cpp.dataflow.new.DataFlow`) and taint-tracking libraries (`semmle.code.cpp.dataflow.new.TaintTracking`) now support tracking flow through static local variables.

cpp/ql/lib/change-notes/released/0.7.3.md

@@ -1,7 +0,0 @@
-## 0.7.3
-
-### Minor Analysis Improvements
-
-* Deleted the deprecated `hasCopyConstructor` predicate from the `Class` class in `Class.qll`.
-* Deleted many deprecated predicates and classes with uppercase `AST`, `SSA`, `CFG`, `API`, etc. in their names. Use the PascalCased versions instead.
-* Deleted the deprecated `CodeDuplication.qll` file.

cpp/ql/lib/change-notes/released/0.7.4.md

@@ -1,3 +0,0 @@
-## 0.7.4
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.8.0.md

@@ -1,9 +0,0 @@
-## 0.8.0
-
-### New Features
-
-* The `ProductFlow::StateConfigSig` signature now includes default predicates for `isBarrier1`, `isBarrier2`, `isAdditionalFlowStep1`, and `isAdditionalFlowStep1`. Hence, it is no longer needed to provide `none()` implementations of these predicates if they are not needed.
-
-### Minor Analysis Improvements
-
-* Deleted the deprecated `getURL` predicate from the `Container`, `Folder`, and `File` classes. Use the `getLocation` predicate instead.

cpp/ql/lib/change-notes/released/0.8.1.md

@@ -1,19 +0,0 @@
-## 0.8.1
-
-### Deprecated APIs
-
-* The library `semmle.code.cpp.dataflow.DataFlow` has been deprecated. Please use `semmle.code.cpp.dataflow.new.DataFlow` instead.
-
-### New Features
-
-* The `DataFlow::StateConfigSig` signature module has gained default implementations for `isBarrier/2` and `isAdditionalFlowStep/4`. 
-  Hence it is no longer needed to provide `none()` implementations of these predicates if they are not needed.
-
-### Minor Analysis Improvements
-
-* Data flow configurations can now include a predicate `neverSkip(Node node)`
-  in order to ensure inclusion of certain nodes in the path explanations. The
-  predicate defaults to the end-points of the additional flow steps provided in
-  the configuration, which means that such steps now always are visible by
-  default in path explanations.
-* The `IRGuards` library has improved handling of pointer addition and subtraction operations.

cpp/ql/lib/change-notes/released/0.9.0.md

@@ -1,14 +0,0 @@
-## 0.9.0
-
-### Breaking Changes
-
-* The `shouldPrintFunction` predicate from `PrintAstConfiguration` has been replaced by `shouldPrintDeclaration`. Users should now override `shouldPrintDeclaration` if they want to limit the declarations that should be printed.
-* The `shouldPrintFunction` predicate from `PrintIRConfiguration` has been replaced by `shouldPrintDeclaration`. Users should now override `shouldPrintDeclaration` if they want to limit the declarations that should be printed.
-
-### Major Analysis Improvements
-
-* The `PrintAST` library now also prints global and namespace variables and their initializers.
-
-### Minor Analysis Improvements
-
-* The `_Float128x` type is no longer exposed as a builtin type. As this type could not occur any code base, this should only affect queries that explicitly looked at the builtin types.

cpp/ql/lib/change-notes/released/0.9.1.md

@@ -1,3 +0,0 @@
-## 0.9.1
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.9.2.md

@@ -1,14 +0,0 @@
-## 0.9.2
-
-### Deprecated APIs
-
-* `getAllocatorCall` on `DeleteExpr` and `DeleteArrayExpr` has been deprecated. `getDeallocatorCall` should be used instead.
-
-### New Features
-
-* Added `DeleteOrDeleteArrayExpr` as a super type of `DeleteExpr` and `DeleteArrayExpr`
-
-### Minor Analysis Improvements
-
-* `delete` and `delete[]` are now modeled as calls to the relevant `operator delete` in the IR. In the case of a dynamic delete call a new instruction `VirtualDeleteFunctionAddress` is used to represent a function that dispatches to the correct delete implementation.
-* Only the 2 level indirection of `argv` (corresponding to `**argv`) is consided for `FlowSource`.

cpp/ql/lib/change-notes/released/0.9.3.md

@@ -1,3 +0,0 @@
-## 0.9.3
-
-No user-facing changes.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.9.3
+lastReleaseVersion: 0.5.0

cpp/ql/lib/experimental/cryptography/Concepts.qll

@@ -1,3 +0,0 @@
-import experimental.cryptography.CryptoArtifact
-import experimental.cryptography.CryptoAlgorithmNames
-import experimental.cryptography.modules.OpenSSL as OpenSSL

cpp/ql/lib/experimental/cryptography/CryptoAlgorithmNames.qll

@@ -1,239 +0,0 @@
-/**
- * Names of known cryptographic algorithms.
- * The names are standardized into upper-case, no spaces, dashes or underscores.
- */
-
-/**
- * Returns a string to represent generally unknown algorithms.
- * Predicate is to be used to get a consistent string representation
- * for unknown algorithms.
- */
-string unknownAlgorithm() { result = "UNKNOWN" }
-
-string getHashType() { result = "HASH" }
-
-string getSymmetricEncryptionType() { result = "SYMMETRIC_ENCRYPTION" }
-
-string getAsymmetricEncryptionType() { result = "ASYMMETRIC_ENCRYPTION" }
-
-string getKeyDerivationType() { result = "KEY_DERIVATION" }
-
-string getCipherBlockModeType() { result = "BLOCK_MODE" }
-
-string getSymmetricPaddingType() { result = "SYMMETRIC_PADDING" }
-
-string getAsymmetricPaddingType() { result = "ASYMMETRIC_PADDING" }
-
-string getEllipticCurveType() { result = "ELLIPTIC_CURVE" }
-
-string getSignatureType() { result = "SIGNATURE" }
-
-string getKeyExchangeType() { result = "KEY_EXCHANGE" }
-
-string getAsymmetricType() {
-  result in [
-      getAsymmetricEncryptionType(), getSignatureType(), getKeyExchangeType(),
-      getEllipticCurveType()
-    ]
-}
-
-predicate isKnownType(string algType) {
-  algType in [
-      getHashType(), getSymmetricEncryptionType(), getAsymmetricEncryptionType(),
-      getKeyDerivationType(), getCipherBlockModeType(), getSymmetricPaddingType(),
-      getAsymmetricPaddingType(), getEllipticCurveType(), getSignatureType(), getKeyExchangeType()
-    ]
-}
-
-predicate isKnownAlgorithm(string name) { isKnownAlgorithm(name, _) }
-
-predicate isKnownAlgorithm(string name, string algType) {
-  isHashingAlgorithm(name) and algType = "HASH"
-  or
-  isEncryptionAlgorithm(name, algType) and
-  algType in ["SYMMETRIC_ENCRYPTION", "ASYMMETRIC_ENCRYPTION"]
-  or
-  isKeyDerivationAlgorithm(name) and algType = "KEY_DERIVATION"
-  or
-  isCipherBlockModeAlgorithm(name) and algType = "BLOCK_MODE"
-  or
-  isPaddingAlgorithm(name, algType) and algType in ["SYMMETRIC_PADDING", "ASYMMETRIC_PADDING"]
-  or
-  isEllipticCurveAlgorithm(name) and algType = "ELLIPTIC_CURVE"
-  or
-  isSignatureAlgorithm(name) and algType = "SIGNATURE"
-  or
-  isKeyExchangeAlgorithm(name) and algType = "KEY_EXCHANGE"
-}
-
-/**
- * Holds if `name` is a known hashing algorithm in the model/library.
- */
-predicate isHashingAlgorithm(string name) {
-  name =
-    [
-      "BLAKE2", "BLAKE2B", "BLAKE2S", "SHA2", "SHA224", "SHA256", "SHA384", "SHA512", "SHA512224",
-      "SHA512256", "SHA3", "SHA3224", "SHA3256", "SHA3384", "SHA3512", "SHAKE128", "SHAKE256",
-      "SM3", "WHIRLPOOL", "POLY1305", "HAVEL128", "MD2", "MD4", "MD5", "PANAMA", "RIPEMD",
-      "RIPEMD128", "RIPEMD256", "RIPEMD160", "RIPEMD320", "SHA0", "SHA1", "SHA", "MGF1", "MGF1SHA1",
-      "MDC2", "SIPHASH"
-    ]
-}
-
-predicate isEncryptionAlgorithm(string name, string algType) {
-  isAsymmetricEncryptionAlgorithm(name) and algType = "ASYMMETRIC_ENCRYPTION"
-  or
-  isSymmetricEncryptionAlgorithm(name) and algType = "SYMMETRIC_ENCRYPTION"
-}
-
-predicate isEncryptionAlgorithm(string name) { isEncryptionAlgorithm(name, _) }
-
-/**
- * Holds if `name` corresponds to a known symmetric encryption algorithm.
- */
-predicate isSymmetricEncryptionAlgorithm(string name) {
-  // NOTE: AES is meant to caputure all possible key lengths
-  name =
-    [
-      "AES", "AES128", "AES192", "AES256", "ARIA", "BLOWFISH", "BF", "ECIES", "CAST", "CAST5",
-      "CAMELLIA", "CAMELLIA128", "CAMELLIA192", "CAMELLIA256", "CHACHA", "CHACHA20",
-      "CHACHA20POLY1305", "GOST", "GOSTR34102001", "GOSTR341094", "GOSTR341194", "GOST2814789",
-      "GOSTR341194", "GOST2814789", "GOST28147", "GOSTR341094", "GOST89", "GOST94", "GOST34102012",
-      "GOST34112012", "IDEA", "RABBIT", "SEED", "SM4", "DES", "DESX", "3DES", "TDES", "2DES",
-      "DES3", "TRIPLEDES", "TDEA", "TRIPLEDEA", "ARC2", "RC2", "ARC4", "RC4", "ARCFOUR", "ARC5",
-      "RC5", "MAGMA", "KUZNYECHIK"
-    ]
-}
-
-/**
- * Holds if `name` corresponds to a known key derivation algorithm.
- */
-predicate isKeyDerivationAlgorithm(string name) {
-  name =
-    [
-      "ARGON2", "CONCATKDF", "CONCATKDFHASH", "CONCATKDFHMAC", "KBKDFCMAC", "BCRYPT", "HKDF",
-      "HKDFEXPAND", "KBKDF", "KBKDFHMAC", "PBKDF1", "PBKDF2", "PBKDF2HMAC", "PKCS5", "SCRYPT",
-      "X963KDF", "EVPKDF"
-    ]
-}
-
-/**
- * Holds if `name` corresponds to a known cipher block mode
- */
-predicate isCipherBlockModeAlgorithm(string name) {
-  name = ["CBC", "GCM", "CCM", "CFB", "OFB", "CFB8", "CTR", "OPENPGP", "XTS", "EAX", "SIV", "ECB"]
-}
-
-/**
- * Holds if `name` corresponds to a known padding algorithm
- */
-predicate isPaddingAlgorithm(string name, string algType) {
-  isSymmetricPaddingAlgorithm(name) and algType = "SYMMETRIC_PADDING"
-  or
-  isAsymmetricPaddingAlgorithm(name) and algType = "ASYMMETRIC_PADDING"
-}
-
-/**
- * holds if `name` corresponds to a known symmetric padding algorithm
- */
-predicate isSymmetricPaddingAlgorithm(string name) { name = ["PKCS7", "ANSIX923"] }
-
-/**
- * Holds if `name` corresponds to a known asymmetric padding algorithm
- */
-predicate isAsymmetricPaddingAlgorithm(string name) { name = ["OAEP", "PKCS1V15", "PSS", "KEM"] }
-
-predicate isBrainpoolCurve(string curveName, int keySize) {
-  // ALL BRAINPOOL CURVES
-  keySize in [160, 192, 224, 256, 320, 384, 512] and
-  (
-    curveName = "BRAINPOOLP" + keySize.toString() + "R1"
-    or
-    curveName = "BRAINPOOLP" + keySize.toString() + "T1"
-  )
-}
-
-predicate isSecCurve(string curveName, int keySize) {
-  // ALL SEC CURVES
-  keySize in [112, 113, 128, 131, 160, 163, 192, 193, 224, 233, 239, 256, 283, 384, 409, 521, 571] and
-  exists(string suff | suff in ["R1", "R2", "K1"] |
-    curveName = "SECT" + keySize.toString() + suff or
-    curveName = "SECP" + keySize.toString() + suff
-  )
-}
-
-predicate isC2Curve(string curveName, int keySize) {
-  // ALL C2 CURVES
-  keySize in [163, 176, 191, 208, 239, 272, 304, 359, 368, 431] and
-  exists(string pre, string suff |
-    pre in ["PNB", "ONB", "TNB"] and suff in ["V1", "V2", "V3", "V4", "V5", "W1", "R1"]
-  |
-    curveName = "C2" + pre + keySize.toString() + suff
-  )
-}
-
-predicate isPrimeCurve(string curveName, int keySize) {
-  // ALL PRIME CURVES
-  keySize in [192, 239, 256] and
-  exists(string suff | suff in ["V1", "V2", "V3"] | curveName = "PRIME" + keySize.toString() + suff)
-}
-
-predicate isEllipticCurveAlgorithm(string curveName) { isEllipticCurveAlgorithm(curveName, _) }
-
-/**
- * Holds if `name` corresponds to a known elliptic curve.
- */
-predicate isEllipticCurveAlgorithm(string curveName, int keySize) {
-  isSecCurve(curveName, keySize)
-  or
-  isBrainpoolCurve(curveName, keySize)
-  or
-  isC2Curve(curveName, keySize)
-  or
-  isPrimeCurve(curveName, keySize)
-  or
-  curveName = "ES256" and keySize = 256
-  or
-  curveName = "CURVE25519" and keySize = 255
-  or
-  curveName = "X25519" and keySize = 255
-  or
-  curveName = "ED25519" and keySize = 255
-  or
-  curveName = "CURVE448" and keySize = 448 // TODO: need to check the key size
-  or
-  curveName = "ED448" and keySize = 448
-  or
-  curveName = "X448" and keySize = 448
-  or
-  curveName = "NUMSP256T1" and keySize = 256
-  or
-  curveName = "NUMSP384T1" and keySize = 384
-  or
-  curveName = "NUMSP512T1" and keySize = 512
-  or
-  curveName = "SM2" and keySize in [256, 512]
-}
-
-/**
- * Holds if `name` corresponds to a known signature algorithm.
- */
-predicate isSignatureAlgorithm(string name) {
-  name =
-    [
-      "DSA", "ECDSA", "EDDSA", "ES256", "ES256K", "ES384", "ES512", "ED25519", "ED448", "ECDSA256",
-      "ECDSA384", "ECDSA512"
-    ]
-}
-
-/**
- * Holds if `name` is a key exchange algorithm.
- */
-predicate isKeyExchangeAlgorithm(string name) {
-  name = ["ECDH", "DH", "DIFFIEHELLMAN", "X25519", "X448"]
-}
-
-/**
- * Holds if `name` corresponds to a known asymmetric encryption.
- */
-predicate isAsymmetricEncryptionAlgorithm(string name) { name = ["RSA"] }

cpp/ql/lib/experimental/cryptography/CryptoArtifact.qll

@@ -1,316 +0,0 @@
-import cpp
-private import experimental.cryptography.CryptoAlgorithmNames
-import semmle.code.cpp.ir.dataflow.TaintTracking
-
-/*
- * A cryptographic artifact is a DataFlow::Node associated with some
- * operation, algorithm, or any other aspect of cryptography.
- */
-
-abstract class CryptographicArtifact extends Expr { }
-
-// /**
-//  * Associates a symmetric encryption algorithm with a block mode.
-//  * The DataFlow::Node representing this association should be the
-//  * point where the algorithm and block mode are combined.
-//  * This may be at the call to encryption or in the construction
-//  * of an object prior to encryption.
-//  */
-// abstract class SymmetricCipher extends CryptographicArtifact{
-//   abstract SymmetricEncryptionAlgorithm getEncryptionAlgorithm();
-//   abstract BlockMode getBlockMode();
-//   final predicate hasBlockMode(){
-//     exists(this.getBlockMode())
-//   }
-// }
-// /**
-//  * A cryptographic operation is a method call that invokes a cryptographic
-//  * algorithm (encrypt/decrypt) or a function in support of a cryptographic algorithm
-//  * (key generation).
-//  *
-//  * Since operations are related to or in support of algorithms, operations must
-//  * provide a reference to their associated algorithm. Often operataions themselves
-//  * encapsulate algorithms, so operations can also extend CryptographicAlgorithm
-//  * and refer to themselves as the target algorithm.
-//  */
-// abstract class CryptographicOperation extends CryptographicArtifact, Call{
-//   // bindingset[paramName, ind]
-//   // final DataFlow::Node getParameterSource(int ind, string paramName){
-//   //   result = Utils::getUltimateSrcFromApiNode(this.(API::CallNode).getParameter(ind, paramName))
-//   // }
-//   final string getAlgorithmName(){
-//     if exists(this.getAlgorithm().getName())
-//     then result = this.getAlgorithm().getName()
-//     else result = unknownAlgorithm()
-//   }
-//   final predicate hasAlgorithm(){
-//     exists(this.getAlgorithm())
-//   }
-//   final predicate isUnknownAlgorithm(){
-//     this.getAlgorithmName() = unknownAlgorithm()
-//     or
-//     not this.hasAlgorithm()
-//   }
-//   // TODO: this might have to be parameterized by a configuration source for
-//   //       situations where an operation is passed an algorithm
-//   abstract CryptographicAlgorithm getAlgorithm();
-// }
-// /** A key generation operation for asymmetric keys */
-// abstract class KeyGen extends CryptographicOperation{
-//   int getAKeySizeInBits(){
-//     result = getKeySizeInBits(_)
-//   }
-//   final predicate hasKeySize(Expr configSrc){
-//     exists(this.getKeySizeInBits(configSrc))
-//   }
-//   final predicate hasKeySize(){
-//     exists(this.getAKeySizeInBits())
-//   }
-//   abstract Expr getKeyConfigSrc();
-//   abstract int getKeySizeInBits(Expr configSrc);
-// }
-abstract class CryptographicOperation extends CryptographicArtifact, Call { }
-
-abstract class KeyGeneration extends CryptographicOperation {
-  // TODO: what if the algorithm is UNKNOWN?
-  abstract Expr getKeyConfigurationSource(CryptographicAlgorithm alg);
-
-  abstract CryptographicAlgorithm getAlgorithm();
-
-  int getKeySizeInBits(CryptographicAlgorithm alg) {
-    result = this.getKeyConfigurationSource(alg).(Literal).getValue().toInt()
-  }
-
-  predicate hasConstantKeySize(CryptographicAlgorithm alg) { exists(this.getKeySizeInBits(alg)) }
-
-  predicate hasKeyConfigurationSource(CryptographicAlgorithm alg) {
-    exists(this.getKeyConfigurationSource(alg))
-  }
-
-  Expr getAKeyConfigurationSource() { result = this.getKeyConfigurationSource(_) }
-}
-
-abstract class AsymmetricKeyGeneration extends KeyGeneration { }
-
-abstract class SymmetricKeyGeneration extends KeyGeneration { }
-
-/**
- * A cryptographic algorithm is a `CryptographicArtifact`
- * representing a cryptographic algorithm (see `CryptoAlgorithmNames.qll`).
- * Cryptographic algorithms can be functions referencing common crypto algorithms (e.g., hashlib.md5)
- * or strings that are used in cryptographic operation configurations (e.g., hashlib.new("md5")).
- * Cryptogrpahic algorithms may also be operations that wrap or abstract one or
- * more algorithms (e.g., cyrptography.fernet.Fernet and AES, CBC and PKCS7).
- *
- * In principle, this class should model the location where an algorithm enters the program, not
- * necessarily where it is used.
- */
-abstract class CryptographicAlgorithm extends CryptographicArtifact {
-  abstract string getName();
-
-  abstract string getAlgType();
-
-  //  string getAlgType(){
-  //   if this instanceof HashAlgorithm then result = getHashType()
-  //   else if this instanceof KeyDerivationAlgorithm then result = getKeyDerivationType()
-  //   else if this instanceof SymmetricEncryptionAlgorithm then result = getSymmetricEncryptionType()
-  //   else if this instanceof AsymmetricEncryptionAlgorithm then result = getAsymmetricEncryptionType()
-  //   else if this instanceof SymmetricEncryptionAlgorithm then result = getSymmetricPaddingType()
-  //   else if this instanceof AsymmetricEncryptionAlgorithm then result = getAsymmetricPaddingType()
-  //   else if this instanceof EllipticCurveAlgorithm then result = getEllipticCurveType()
-  //   else if this instanceof BlockMode then result = getCipherBlockModeType()
-  //   else if this instanceof KeyExchangeAlgorithm then result = getKeyExchangeType()
-  //   else if this instanceof SigningAlgorithm then result = getSignatureType()
-  //   else result = unknownAlgorithm()
-  // }
-  // TODO: handle case where name isn't known, not just unknown?
-  /**
-   * Normalizes a raw name into a normalized name as found in `CryptoAlgorithmNames.qll`.
-   * Subclassess should override for more api-specific normalization.
-   * By deafult, converts a raw name to upper-case with no hyphen, underscore, hash, or space.
-   */
-  bindingset[s]
-  string normalizeName(string s) {
-    exists(string normStr | normStr = s.toUpperCase().regexpReplaceAll("[-_ ]|/", "") |
-      result = normStr and isKnownAlgorithm(result)
-      or
-      result = unknownAlgorithm() and not isKnownAlgorithm(normStr)
-    )
-  }
-
-  abstract Expr configurationSink();
-
-  predicate hasConfigurationSink() { exists(this.configurationSink()) }
-}
-
-abstract class HashAlgorithm extends CryptographicAlgorithm {
-  final string getHashName() {
-    if exists(string n | n = this.getName() and isHashingAlgorithm(n))
-    then isHashingAlgorithm(result) and result = this.getName()
-    else result = unknownAlgorithm()
-  }
-
-  override string getAlgType() { result = getHashType() }
-}
-
-abstract class KeyDerivationAlgorithm extends CryptographicAlgorithm {
-  final string getKDFName() {
-    if exists(string n | n = this.getName() and isKeyDerivationAlgorithm(n))
-    then isKeyDerivationAlgorithm(result) and result = this.getName()
-    else result = unknownAlgorithm()
-  }
-
-  override string getAlgType() { result = getKeyDerivationType() }
-}
-
-// abstract class KeyDerivationOperation extends CryptographicOperation{
-//   DataFlow::Node getIterationSizeSrc(){
-//     none()
-//   }
-//   DataFlow::Node getSaltConfigSrc(){
-//     none()
-//   }
-//   DataFlow::Node getHashConfigSrc(){
-//     none()
-//   }
-//   // TODO: get encryption algorithm for CBC-based KDF?
-//   DataFlow::Node getDerivedKeySizeSrc(){
-//     none()
-//   }
-//   DataFlow::Node getModeSrc(){
-//     none()
-//   }
-//   // TODO: add more to cover all the parameters of most KDF operations? Perhaps subclass for each type?
-//   abstract predicate requiresIteration();
-//   abstract predicate requiresSalt();
-//   abstract predicate requiresHash();
-//   //abstract predicate requiresKeySize(); // Going to assume all requires a size
-//   abstract predicate requiresMode();
-// }
-abstract class EncryptionAlgorithm extends CryptographicAlgorithm {
-  final predicate isAsymmetric() { this instanceof AsymmetricEncryptionAlgorithm }
-
-  final predicate isSymmetric() { not this.isAsymmetric() }
-  // NOTE: DO_NOT add getEncryptionName here, we rely on the fact the parent
-  //       class does not have this common predicate.
-}
-
-/**
- * A parent class to represent any algorithm for which
- * asymmetric cryptography is involved.
- * Intended to be distinct from AsymmetricEncryptionAlgorithm
- * which is intended only for asymmetric algorithms that specifically encrypt.
- */
-abstract class AsymmetricAlgorithm extends CryptographicAlgorithm { }
-
-/**
- * Algorithms directly or indirectly related to asymmetric encryption,
- * e.g., RSA, DSA, but also RSA padding algorithms
- */
-abstract class AsymmetricEncryptionAlgorithm extends AsymmetricAlgorithm, EncryptionAlgorithm {
-  final string getEncryptionName() {
-    if exists(string n | n = this.getName() and isAsymmetricEncryptionAlgorithm(n))
-    then isAsymmetricEncryptionAlgorithm(result) and result = this.getName()
-    else result = unknownAlgorithm()
-  }
-
-  override string getAlgType() { result = getAsymmetricEncryptionType() }
-}
-
-/**
- * Algorithms directly or indirectly related to symmetric encryption,
- * e.g., AES, DES, but also block modes and padding
- */
-abstract class SymmetricEncryptionAlgorithm extends EncryptionAlgorithm {
-  final string getEncryptionName() {
-    if exists(string n | n = this.getName() and isSymmetricEncryptionAlgorithm(n))
-    then isSymmetricEncryptionAlgorithm(result) and result = this.getName()
-    else result = unknownAlgorithm()
-  }
-
-  // TODO: add a stream cipher predicate?
-  override string getAlgType() { result = getSymmetricEncryptionType() }
-}
-
-// Used only to categorize all padding into a single object,
-// DO_NOT add predicates here. Only for categorization purposes.
-abstract class PaddingAlgorithm extends CryptographicAlgorithm { }
-
-abstract class SymmetricPadding extends PaddingAlgorithm {
-  final string getPaddingName() {
-    if exists(string n | n = this.getName() and isSymmetricPaddingAlgorithm(n))
-    then isSymmetricPaddingAlgorithm(result) and result = this.getName()
-    else result = unknownAlgorithm()
-  }
-
-  override string getAlgType() { result = getSymmetricPaddingType() }
-}
-
-abstract class AsymmetricPadding extends PaddingAlgorithm {
-  final string getPaddingName() {
-    if exists(string n | n = this.getName() and isAsymmetricPaddingAlgorithm(n))
-    then isAsymmetricPaddingAlgorithm(result) and result = this.getName()
-    else result = unknownAlgorithm()
-  }
-
-  override string getAlgType() { result = getAsymmetricPaddingType() }
-}
-
-abstract class EllipticCurveAlgorithm extends AsymmetricAlgorithm {
-  final string getCurveName() {
-    if exists(string n | n = this.getName() and isEllipticCurveAlgorithm(n))
-    then isEllipticCurveAlgorithm(result) and result = this.getName()
-    else result = unknownAlgorithm()
-  }
-
-  final int getCurveBitSize() { isEllipticCurveAlgorithm(this.getCurveName(), result) }
-
-  override string getAlgType() { result = getEllipticCurveType() }
-}
-
-abstract class BlockModeAlgorithm extends CryptographicAlgorithm {
-  final string getBlockModeName() {
-    if exists(string n | n = this.getName() and isCipherBlockModeAlgorithm(n))
-    then isCipherBlockModeAlgorithm(result) and result = this.getName()
-    else result = unknownAlgorithm()
-  }
-
-  /**
-   * Gets the source of the IV configuration.
-   */
-  abstract Expr getIVorNonce();
-
-  final predicate hasIVorNonce() { exists(this.getIVorNonce()) }
-
-  override string getAlgType() { result = getCipherBlockModeType() }
-}
-
-// abstract class KeyWrapOperation extends CryptographicOperation{
-// }
-abstract class AuthenticatedEncryptionAlgorithm extends SymmetricEncryptionAlgorithm {
-  final string getAuthticatedEncryptionName() {
-    if exists(string n | n = this.getName() and isSymmetricEncryptionAlgorithm(n))
-    then isSymmetricEncryptionAlgorithm(result) and result = this.getName()
-    else result = unknownAlgorithm()
-  }
-}
-
-abstract class KeyExchangeAlgorithm extends AsymmetricAlgorithm {
-  final string getKeyExchangeName() {
-    if exists(string n | n = this.getName() and isKeyExchangeAlgorithm(n))
-    then isKeyExchangeAlgorithm(result) and result = this.getName()
-    else result = unknownAlgorithm()
-  }
-
-  override string getAlgType() { result = getKeyExchangeType() }
-}
-
-abstract class SigningAlgorithm extends AsymmetricAlgorithm {
-  final string getSigningName() {
-    if exists(string n | n = this.getName() and isSignatureAlgorithm(n))
-    then isSignatureAlgorithm(result) and result = this.getName()
-    else result = unknownAlgorithm()
-  }
-
-  override string getAlgType() { result = getSignatureType() }
-}

cpp/ql/lib/experimental/cryptography/modules/OpenSSL.qll

@@ -1,718 +0,0 @@
-import cpp
-import experimental.cryptography.CryptoAlgorithmNames
-import experimental.cryptography.CryptoArtifact
-import experimental.cryptography.utils.OpenSSL.CryptoFunction
-import experimental.cryptography.utils.OpenSSL.AlgorithmSink
-import experimental.cryptography.utils.OpenSSL.PassthroughFunction
-import experimental.cryptography.utils.OpenSSL.CryptoAlgorithm
-import experimental.cryptography.CryptoArtifact
-// import semmle.code.cpp.ir.dataflow.TaintTracking
-import semmle.code.cpp.ir.dataflow.DataFlow
-
-/**
- *  Problematic case in OpenSSL speed.c
- *    static const char *names[ALGOR_NUM] = {
- *        "md2", "mdc2", "md4", "md5", "sha1", "rmd160",
- *        "sha256", "sha512", "whirlpool", "hmac(md5)",
- *        "des-cbc", "des-ede3", "rc4", "idea-cbc", "seed-cbc",
- *        "rc2-cbc", "rc5-cbc", "blowfish", "cast-cbc",
- *        "aes-128-cbc", "aes-192-cbc", "aes-256-cbc",
- *        "camellia-128-cbc", "camellia-192-cbc", "camellia-256-cbc",
- *        "evp", "ghash", "rand", "cmac"
- *    };
- *
- *    Every entry is considered a block mode, hash, and symmetric encryption algorithm
- *    getEncryptionName for example, will return unknown
- */
-predicate nodeToExpr(DataFlow::Node node, Expr e) {
-  e = node.asExpr() or e = node.asIndirectArgument()
-}
-
-Expr getExprFromNode(DataFlow::Node node) { nodeToExpr(node, result) }
-
-DataFlow::Node getNodeFromExpr(Expr e) { nodeToExpr(result, e) }
-
-predicate isEVP_PKEY_CTX(Type t) { t.getUnderlyingType().stripType().getName() = "evp_pkey_ctx_st" }
-
-/**
- * An expression representing an EVP_PKEY_CTX* at the location of a
- * known AlgorithmSinkArgument.
- * The EVP_PKEY_CTX* represents the location where the CTX is tied to the algorithm,
- * and can be used as a source for tracing EVP_PKEY_CTX to other operations.
- */
-class Known_EVP_PKEY_CTX_Ptr_Source extends Expr {
-  Known_EVP_PKEY_CTX_Ptr_Source() {
-    isEVP_PKEY_CTX(this.getUnderlyingType()) and
-    this.getUnderlyingType() instanceof PointerType and
-    exists(AlgorithmSinkArgument arg, Call sinkCall |
-      arg.getSinkCall() = sinkCall and
-      sinkCall.getAnArgument() = this
-      or
-      this = sinkCall
-    )
-  }
-}
-
-// module CTXFlow implements DataFlow::ConfigSig{
-//     predicate isSource(DataFlow::Node source) {
-//         // ASSUMPTION: at a sink, an algorithm is converted into a CTX through a return of the call only
-//         //             and is the primary source of interest for CTX tracing
-//         source.asExpr() instanceof AlgorithmSinkArgument
-//     }
-//     predicate isSink(DataFlow::Node sink){
-//         sink.asExpr() instanceof CTXSink
-//     }
-//     predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
-//         //  cls.getName() = "asn1_object_st" flow out on any EVP_PKEY_CTX which is "evp_pkey_ctx_st"
-//         exists(Call c |
-//             isEVP_PKEY_CTX(c.getUnderlyingType()) and
-//             node1.asExpr() = c.getAnArgument() and c = node2.asExpr())
-//     }
-// }
-// module CTXFlowConfig = DataFlow::Global<CTXFlow>;
-// TODO: currently only handles tracing from literals to sinks
-module LiteralAlgorithmTracerConfig implements DataFlow::ConfigSig {
-  predicate isSource(DataFlow::Node source) {
-    source.asExpr() instanceof Literal and
-    // Optimization to reduce literal tracing on integers to only those that are known/relevant NIDs.
-    (
-      exists(source.asExpr().getValue().toInt())
-      implies
-      source.asExpr().getValue().toInt() < getNIDMax()
-    ) and
-    // False positives observed inside OBJ_nid2* and OBJ_sn2* functions where NULL is a possible assignment.
-    // While this is a concern, it only occurs if the object being referenced is NULL to begin with
-    // Perhaps a different query should be used to find these caes if they represent a threat.
-    // Filter out any open ssl function source in a function namae Obj_*
-    // False positives in OpenSSL also observed for CRYPTO_strndup (filtering any CRYPTO_* function)
-    // due to setting a null byte in the string
-    (
-      isPossibleOpenSSLFunction(source.getEnclosingCallable())
-      implies
-      (
-        not source.getEnclosingCallable().getName().matches("OBJ_%") and
-        not source.getEnclosingCallable().getName().matches("CRYPTO_%")
-      )
-    )
-  }
-
-  predicate isSink(DataFlow::Node sink) {
-    // A sink is a call to a function that takes an algorithm as an argument
-    // must include checks for asIndirectArgument since the input may be a pointer to an object
-    // and the member of the object holds the algorithm on the trace.
-    getExprFromNode(sink) instanceof AlgorithmSinkArgument
-  }
-
-  predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
-    knownPassThroughStep(node1, node2)
-  }
-
-  predicate isBarrier(DataFlow::Node node) {
-    // If the node is the 'next' argument of a isCallPassThrough, it is only allowed if it is an out parameter
-    // i.e., a defining argument. This barrier says that if the node is an expression not an out parameter, it is filtered.
-    // Out arguments will not be filtered.
-    exists(Call c | knownPassthoughCall(c, _, node.asExpr()) and c.getAnArgument() = node.asExpr())
-    or
-    // False positive reducer, don't flow out through argv
-    node.asVariable().hasName("argv")
-    or
-    node.asIndirectVariable().hasName("argv")
-  }
-
-  predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c) {
-    // Assume a read on crypto identifying field for any object of type asn1_object_st (i.e., ASN1_OBJECT)
-    exists(Class cls | cls.getName() = "asn1_object_st" |
-      node.getType().getUnspecifiedType().stripType() = cls and
-      c.(DataFlow::FieldContent).getField() = cls.getAMember() and
-      c.(DataFlow::FieldContent).getField().getName() in ["nid", "sn", "ln"]
-    )
-  }
-}
-
-module LiteralAlgorithmTracer = DataFlow::Global<LiteralAlgorithmTracerConfig>;
-
-/**
- * `source` is an expression that is a source of an algorithm of type `algType`.
- * `algType` may be `UNKONWN`.
- *  See CryptoAlgorithmNames for other possible values of `algType`.
- */
-bindingset[sinkAlgType]
-predicate hasLiteralPathToAlgSink(DataFlow::Node source, DataFlow::Node sink, string sinkAlgType) {
-  LiteralAlgorithmTracer::flow(source, sink) and
-  getExprFromNode(sink).(AlgorithmSinkArgument).algType() = sinkAlgType
-}
-
-private predicate knownTracedAlgorithm(Literal e, string srcSinkType) {
-  knownTracedAlgorithm(e, srcSinkType, srcSinkType)
-}
-
-private predicate knownTracedAlgorithm(Literal e, string srcType, string sinkType) {
-  resolveAlgorithmFromLiteral(e, _, srcType) and
-  hasLiteralPathToAlgSink(DataFlow::exprNode(e), _, sinkType) and
-  isKnownType(sinkType) and
-  isKnownType(srcType)
-}
-
-private predicate unknownTracedLiteralAlgorithm(Literal e, string srcSinkType) {
-  // Asymmetric special case:
-  // Since asymmetric algorithm sinks are used for various categories of asymmetric algorithms
-  // an asymmetric algorithm is only unknown if there is no trace from any asymmetric type to the given srcSinkType sink
-  if getAsymmetricType() = srcSinkType
-  then forall(string t | t = getAsymmetricType() | unknownTracedLiteralAlgorithm(e, t, srcSinkType))
-  else unknownTracedLiteralAlgorithm(e, srcSinkType, srcSinkType)
-}
-
-private predicate unknownTracedLiteralAlgorithm(Literal e, string srcType, string sinkType) {
-  // the literal resolves to an algorithm, but not to the sinktype
-  // or generally doesn't resolve to any algorithm type
-  // this case covers 'nonsense' cases e.g., use RSA for symmetric encryption
-  not resolveAlgorithmFromLiteral(e, _, srcType) and
-  isValidAlgorithmLiteral(e) and
-  hasLiteralPathToAlgSink(DataFlow::exprNode(e), _, sinkType) and
-  isKnownType(sinkType) and
-  isKnownType(srcType)
-}
-
-private predicate unknownTracedNonLiteralAlgorithm(AlgorithmSinkArgument e, string srcSinkType) {
-  // Asymmetric special case:
-  // Since asymmetric algorithm sinks are used for various categories of asymmetric algorithms
-  // an asymmetric algorithm is only unknown if there is no trace from any asymmetric type to the given srcSinkType sink
-  if getAsymmetricType() = srcSinkType
-  then
-    forall(string t | t = getAsymmetricType() | unknownTracedNonLiteralAlgorithm(e, t, srcSinkType))
-  else unknownTracedNonLiteralAlgorithm(e, srcSinkType, srcSinkType)
-}
-
-private predicate unknownTracedNonLiteralAlgorithm(
-  AlgorithmSinkArgument e, string srcType, string sinkType
-) {
-  not hasLiteralPathToAlgSink(_, getNodeFromExpr(e), srcType) and
-  LiteralAlgorithmTracerConfig::isSink(getNodeFromExpr(e)) and
-  e.algType() = sinkType and
-  isKnownType(srcType) and
-  isKnownType(sinkType)
-}
-
-private predicate functionAlgorithm(Call c, string algType) {
-  isOpenSSLCryptoFunctionCall(c, _, algType)
-}
-
-abstract class OpenSSLTracedAlgorithm extends CryptographicAlgorithm {
-  override string getName() { resolveAlgorithmFromLiteral(this, result, this.getAlgType()) }
-
-  override Expr configurationSink() {
-    exists(DataFlow::Node sink |
-      hasLiteralPathToAlgSink(DataFlow::exprNode(this), sink, this.getAlgType())
-    |
-      result = getExprFromNode(sink)
-    )
-  }
-}
-
-abstract class OpenSSLFunctionAlgorithm extends CryptographicAlgorithm {
-  override string getName() { isOpenSSLCryptoFunctionCall(this, result, this.getAlgType()) }
-
-  override Expr configurationSink() { result = this }
-}
-
-abstract class OpenSSLUnknownTracedLiteralAlgorithm extends CryptographicAlgorithm {
-  override string getName() { result = unknownAlgorithm() }
-
-  override Expr configurationSink() {
-    exists(DataFlow::Node sink |
-      hasLiteralPathToAlgSink(DataFlow::exprNode(this), sink, this.getAlgType())
-    |
-      result = getExprFromNode(sink)
-    )
-  }
-}
-
-abstract class OpenSSLUnknownTracedNonLiteralAlgorithm extends CryptographicAlgorithm {
-  override string getName() { result = unknownAlgorithm() }
-
-  override Expr configurationSink() { result = this }
-}
-
-module SymmetricEncryption {
-  abstract class OpenSSLSymmetricEncryptionAlgorithm extends SymmetricEncryptionAlgorithm { }
-
-  class OpenSSLSymmetricEncryptionTracedAlgorithm extends OpenSSLTracedAlgorithm,
-    OpenSSLSymmetricEncryptionAlgorithm
-  {
-    OpenSSLSymmetricEncryptionTracedAlgorithm() {
-      knownTracedAlgorithm(this, getSymmetricEncryptionType())
-    }
-  }
-
-  class OpenSSLSymmetricEncryptionFunctionAlgorithm extends OpenSSLFunctionAlgorithm,
-    OpenSSLSymmetricEncryptionAlgorithm
-  {
-    OpenSSLSymmetricEncryptionFunctionAlgorithm() {
-      functionAlgorithm(this, getSymmetricEncryptionType())
-    }
-  }
-
-  class OpenSSLSymmetricEncryptionTracedUnknownLiteralAlgorithm extends OpenSSLUnknownTracedLiteralAlgorithm,
-    OpenSSLSymmetricEncryptionAlgorithm
-  {
-    OpenSSLSymmetricEncryptionTracedUnknownLiteralAlgorithm() {
-      unknownTracedLiteralAlgorithm(this, getSymmetricEncryptionType())
-    }
-  }
-
-  class OpenSSLSymmetricEncryptionUnknownNonLiteralTracedAlgorithm extends OpenSSLUnknownTracedNonLiteralAlgorithm,
-    OpenSSLSymmetricEncryptionAlgorithm
-  {
-    OpenSSLSymmetricEncryptionUnknownNonLiteralTracedAlgorithm() {
-      unknownTracedNonLiteralAlgorithm(this, getSymmetricEncryptionType())
-    }
-  }
-}
-
-module BlockModes {
-  /**
-   * In OpenSSL, block modes are associated directly with symmetric encryption algorithms.
-   * As such, OpenSSLBLockModes are modeled as extensions of any openssl symmetric encryption algorithm
-   */
-  class OpenSSLBlockModeAlgorithm extends BlockModeAlgorithm, Expr instanceof SymmetricEncryption::OpenSSLSymmetricEncryptionAlgorithm
-  {
-    OpenSSLBlockModeAlgorithm() {
-      //two cases, either the block mode is a literal or it is a function call
-      resolveAlgorithmFromLiteral(this, _, "BLOCK_MODE")
-      or
-      isOpenSSLCryptoFunctionCall(this, _, "BLOCK_MODE")
-    }
-
-    override string getName() {
-      resolveAlgorithmFromLiteral(this, result, "BLOCK_MODE")
-      or
-      isOpenSSLCryptoFunctionCall(this, result, "BLOCK_MODE")
-    }
-
-    override Expr configurationSink() {
-      result = this.(SymmetricEncryption::OpenSSLSymmetricEncryptionAlgorithm).configurationSink()
-    }
-
-    override Expr getIVorNonce() {
-      // TODO
-      none()
-    }
-  }
-
-  class UnknownOpenSSLBlockModeAlgorithm extends BlockModeAlgorithm, Expr instanceof SymmetricEncryption::OpenSSLSymmetricEncryptionAlgorithm
-  {
-    UnknownOpenSSLBlockModeAlgorithm() {
-      //two cases, either the block mode is a literal or it is a function call
-      not resolveAlgorithmFromLiteral(this, _, "BLOCK_MODE") and
-      not isOpenSSLCryptoFunctionCall(this, _, "BLOCK_MODE")
-    }
-
-    override string getName() { result = unknownAlgorithm() }
-
-    override Expr configurationSink() {
-      result = this.(SymmetricEncryption::OpenSSLSymmetricEncryptionAlgorithm).configurationSink()
-    }
-
-    override Expr getIVorNonce() { none() }
-  }
-}
-
-module Hashes {
-  abstract class OpenSSLHashAlgorithm extends HashAlgorithm { }
-
-  class OpenSSLHashTracedAlgorithm extends OpenSSLTracedAlgorithm, OpenSSLHashAlgorithm {
-    OpenSSLHashTracedAlgorithm() { knownTracedAlgorithm(this, getHashType()) }
-  }
-
-  class OpenSSLHashFunctionAlgorithm extends OpenSSLFunctionAlgorithm, OpenSSLHashAlgorithm {
-    OpenSSLHashFunctionAlgorithm() { functionAlgorithm(this, getHashType()) }
-  }
-
-  class OpenSSLHashTracedUnknownLiteralAlgorithm extends OpenSSLUnknownTracedLiteralAlgorithm,
-    OpenSSLHashAlgorithm
-  {
-    OpenSSLHashTracedUnknownLiteralAlgorithm() {
-      unknownTracedLiteralAlgorithm(this, getHashType())
-    }
-  }
-
-  class OpenSSLHashUnknownNonLiteralTracedAlgorithm extends OpenSSLUnknownTracedNonLiteralAlgorithm,
-    OpenSSLHashAlgorithm
-  {
-    OpenSSLHashUnknownNonLiteralTracedAlgorithm() {
-      unknownTracedNonLiteralAlgorithm(this, getHashType())
-    }
-  }
-
-  class OpenSSLNullHash extends HashAlgorithm {
-    OpenSSLNullHash() {
-      exists(Call c |
-        this = c and
-        isPossibleOpenSSLFunction(c.getTarget()) and
-        c.getTarget().getName() in ["EVP_md_null"]
-      )
-    }
-
-    override string getName() { result = unknownAlgorithm() }
-
-    override Expr configurationSink() { result = this }
-  }
-}
-
-module EllipticCurves {
-  // TODO: need to address EVP_PKEY_Q_keygen where the type is "EC" but the curve is UNKNOWN?
-  class OpenSSLEllipticCurveTracedAlgorithm extends OpenSSLTracedAlgorithm, EllipticCurveAlgorithm {
-    OpenSSLEllipticCurveTracedAlgorithm() { knownTracedAlgorithm(this, getEllipticCurveType()) }
-  }
-
-  class OpenSSLEllipticCurveFunctionAlgorithm extends OpenSSLFunctionAlgorithm,
-    EllipticCurveAlgorithm
-  {
-    OpenSSLEllipticCurveFunctionAlgorithm() { functionAlgorithm(this, getEllipticCurveType()) }
-  }
-
-  class OpenSSLEllipticCurveTracedUnknownLiteralAlgorithm extends OpenSSLUnknownTracedLiteralAlgorithm,
-    EllipticCurveAlgorithm
-  {
-    OpenSSLEllipticCurveTracedUnknownLiteralAlgorithm() {
-      unknownTracedLiteralAlgorithm(this, getEllipticCurveType())
-    }
-  }
-
-  class OpenSSLEllipticCurvehUnknownNonLiteralTracedAlgorithm extends OpenSSLUnknownTracedNonLiteralAlgorithm,
-    EllipticCurveAlgorithm
-  {
-    OpenSSLEllipticCurvehUnknownNonLiteralTracedAlgorithm() {
-      unknownTracedNonLiteralAlgorithm(this, getEllipticCurveType())
-    }
-  }
-
-  // https://www.openssl.org/docs/manmaster/man3/EC_KEY_new_ex.html
-  class OpenSSLNullEllipticCurve extends EllipticCurveAlgorithm {
-    OpenSSLNullEllipticCurve() {
-      exists(Call c |
-        this = c and
-        isPossibleOpenSSLFunction(c.getTarget()) and
-        c.getTarget().getName() in ["EC_KEY_new", "EC_KEY_new_ex"]
-      )
-    }
-
-    override string getName() { result = unknownAlgorithm() }
-
-    override Expr configurationSink() { result = this }
-  }
-}
-
-module AsymmetricEncryption {
-  class OpenSSLAsymmetricEncryptionTracedAlgorithm extends OpenSSLTracedAlgorithm,
-    AsymmetricEncryptionAlgorithm
-  {
-    OpenSSLAsymmetricEncryptionTracedAlgorithm() {
-      knownTracedAlgorithm(this, getAsymmetricEncryptionType())
-    }
-  }
-
-  class OpenSSLAsymmetricEncryptionFunctionAlgorithm extends OpenSSLFunctionAlgorithm,
-    AsymmetricEncryptionAlgorithm
-  {
-    OpenSSLAsymmetricEncryptionFunctionAlgorithm() {
-      functionAlgorithm(this, getAsymmetricEncryptionType())
-    }
-  }
-
-  class OpenSSLAsymmetricEncryptionTracedUnknownLiteralAlgorithm extends OpenSSLUnknownTracedLiteralAlgorithm,
-    AsymmetricEncryptionAlgorithm
-  {
-    OpenSSLAsymmetricEncryptionTracedUnknownLiteralAlgorithm() {
-      unknownTracedLiteralAlgorithm(this, getAsymmetricEncryptionType())
-    }
-  }
-
-  class OpenSSLAsymmetricEncryptionUnknownNonLiteralTracedAlgorithm extends OpenSSLUnknownTracedNonLiteralAlgorithm,
-    AsymmetricEncryptionAlgorithm
-  {
-    OpenSSLAsymmetricEncryptionUnknownNonLiteralTracedAlgorithm() {
-      unknownTracedNonLiteralAlgorithm(this, getAsymmetricEncryptionType())
-    }
-  }
-}
-
-module SigningAlgorithms {
-  class OpenSSLSignatureTracedAlgorithm extends OpenSSLTracedAlgorithm, SigningAlgorithm {
-    OpenSSLSignatureTracedAlgorithm() { knownTracedAlgorithm(this, getSignatureType()) }
-  }
-
-  class OpenSSLSignatureFunctionAlgorithm extends OpenSSLFunctionAlgorithm, SigningAlgorithm {
-    OpenSSLSignatureFunctionAlgorithm() { functionAlgorithm(this, getSignatureType()) }
-  }
-
-  class OpenSSLSignatureTracedUnknownLiteralAlgorithm extends OpenSSLUnknownTracedLiteralAlgorithm,
-    SigningAlgorithm
-  {
-    OpenSSLSignatureTracedUnknownLiteralAlgorithm() {
-      unknownTracedLiteralAlgorithm(this, getSignatureType())
-    }
-  }
-
-  class OpenSSLSignatureUnknownNonLiteralTracedAlgorithm extends OpenSSLUnknownTracedNonLiteralAlgorithm,
-    SigningAlgorithm
-  {
-    OpenSSLSignatureUnknownNonLiteralTracedAlgorithm() {
-      unknownTracedNonLiteralAlgorithm(this, getSignatureType())
-    }
-  }
-}
-
-module KeyExchange {
-  class OpenSSLKeyExchangeTracedAlgorithm extends OpenSSLTracedAlgorithm, KeyExchangeAlgorithm {
-    OpenSSLKeyExchangeTracedAlgorithm() { knownTracedAlgorithm(this, getKeyExchangeType()) }
-  }
-
-  class OpenSSLKeyExchangeFunctionAlgorithm extends OpenSSLFunctionAlgorithm, KeyExchangeAlgorithm {
-    OpenSSLKeyExchangeFunctionAlgorithm() { functionAlgorithm(this, getKeyExchangeType()) }
-  }
-
-  class OpenSSLKeyExchangeTracedUnknownLiteralAlgorithm extends OpenSSLUnknownTracedLiteralAlgorithm,
-    KeyExchangeAlgorithm
-  {
-    OpenSSLKeyExchangeTracedUnknownLiteralAlgorithm() {
-      unknownTracedLiteralAlgorithm(this, getKeyExchangeType())
-    }
-  }
-
-  class OpenSSLKeyExchangeUnknownNonLiteralTracedAlgorithm extends OpenSSLUnknownTracedNonLiteralAlgorithm,
-    KeyExchangeAlgorithm
-  {
-    OpenSSLKeyExchangeUnknownNonLiteralTracedAlgorithm() {
-      unknownTracedNonLiteralAlgorithm(this, getKeyExchangeType())
-    }
-  }
-}
-
-module KeyGeneration {
-  /**
-   * Functions that explicitly set key generation parameters.
-   * `sizeInd` is the parameter specifying the size of the key.
-   * `outInd` is the parameter or return value that the key is written to.
-   * `outInd` is -1 if the key is written to the return value.
-   */
-  predicate isAsymmetricKeyGenExplicitAlgorithm(Function func, int sizeInd, int outInd) {
-    isPossibleOpenSSLFunction(func) and
-    exists(string name | func.hasGlobalName(name) |
-      name in [
-          "EVP_PKEY_CTX_set_dsa_paramgen_bits", "DSA_generate_parameters_ex",
-          "EVP_PKEY_CTX_set_rsa_keygen_bits", "RSA_generate_key_ex", "RSA_generate_key_fips",
-          "EVP_PKEY_CTX_set_dh_paramgen_prime_len", "DH_generate_parameters_ex"
-        ] and
-      sizeInd = 1 and
-      outInd = 0
-      or
-      name in ["DSA_generate_parameters", "RSA_generate_key", "DH_generate_parameters"] and
-      sizeInd = 0 and
-      outInd = -1
-    ) and
-    exists(Type t |
-      (
-        if sizeInd = -1
-        then t = func.getType().getUnderlyingType()
-        else t = func.getParameter(sizeInd).getUnderlyingType()
-      ) and
-      t instanceof IntegralType and
-      not t instanceof CharType
-    )
-  }
-
-  module AsymExplicitAlgKeyLengthFlowConfig implements DataFlow::ConfigSig {
-    predicate isSource(DataFlow::Node node) {
-      // Optimizations to avoid tracing all integers
-      node.asExpr().(Literal).getValue().toInt() > 0 and // exclude sentinel values
-      node.asExpr().(Literal).getValue().toInt() < 8500
-    }
-
-    predicate isSink(DataFlow::Node node) {
-      exists(FunctionCall c, int sizeInd |
-        isAsymmetricKeyGenExplicitAlgorithm(c.getTarget(), sizeInd, _) and
-        c.getArgument(sizeInd) = node.asExpr()
-      )
-    }
-  }
-
-  module AsymExplicitAlgKeyLengthFlow = DataFlow::Global<AsymExplicitAlgKeyLengthFlowConfig>;
-
-  class OpenSSLAsymmetricKeyGenTiedToAlgorithm extends AsymmetricKeyGeneration {
-    OpenSSLAsymmetricKeyGenTiedToAlgorithm() {
-      exists(Call c |
-        this = c and
-        isPossibleOpenSSLFunction(c.getTarget()) and
-        isAsymmetricKeyGenExplicitAlgorithm(c.getTarget(), _, _)
-      )
-    }
-
-    override CryptographicAlgorithm getAlgorithm() { result = this }
-
-    override Expr getKeyConfigurationSource(CryptographicAlgorithm alg) {
-      alg = this and
-      exists(int sizeInd |
-        isAsymmetricKeyGenExplicitAlgorithm(this.getTarget(), sizeInd, _) and
-        AsymExplicitAlgKeyLengthFlow::flow(DataFlow::exprNode(result),
-          DataFlow::exprNode(this.getArgument(sizeInd)))
-      )
-    }
-  }
-
-  module Length_to_RSA_EVP_PKEY_Q_keygen_Config implements DataFlow::ConfigSig {
-    predicate isSource(DataFlow::Node node) {
-      // Optimizations to avoid tracing all integers
-      node.asExpr().(Literal).getValue().toInt() > 0 and // exclude sentinel values
-      node.asExpr().(Literal).getValue().toInt() < 5000
-    }
-
-    predicate isSink(DataFlow::Node node) {
-      exists(FunctionCall c |
-        c.getTarget().getName() = "EVP_PKEY_Q_keygen" and
-        isPossibleOpenSSLFunction(c.getTarget()) and
-        c.getArgument(3) = node.asExpr()
-      )
-    }
-  }
-
-  module Length_to_RSA_EVP_PKEY_Q_keygen_Flow =
-    DataFlow::Global<Length_to_RSA_EVP_PKEY_Q_keygen_Config>;
-
-  class OpenSSL_RSA_EVP_PKEY_Q_keygen extends AsymmetricKeyGeneration {
-    OpenSSL_RSA_EVP_PKEY_Q_keygen() {
-      exists(Call c |
-        this = c and
-        isPossibleOpenSSLFunction(c.getTarget()) and
-        this.getTarget().getName() = "EVP_PKEY_Q_keygen" and
-        this.getArgument(3).getUnderlyingType() instanceof IntegralType
-      )
-    }
-
-    override CryptographicAlgorithm getAlgorithm() {
-      result.configurationSink().(AlgorithmSinkArgument).getSinkCall() = this
-    }
-
-    override Expr getKeyConfigurationSource(CryptographicAlgorithm alg) {
-      alg = this.getAlgorithm() and
-      Length_to_RSA_EVP_PKEY_Q_keygen_Flow::flow(DataFlow::exprNode(result),
-        DataFlow::exprNode(this.getArgument(3)))
-    }
-  }
-
-  predicate isKeyGenOperationWithNoSize(Function func) {
-    isPossibleOpenSSLFunction(func) and
-    exists(string name | func.hasGlobalName(name) |
-      name in ["EVP_PKEY_keygen", "DSA_generate_key", "DH_generate_key", "EVP_PKEY_generate"]
-    )
-  }
-
-  module KeyGenKeySizeInitToKeyGenConfig implements DataFlow::ConfigSig {
-    predicate isSource(DataFlow::Node node) {
-      exists(Call c, Function func, int outInd |
-        isAsymmetricKeyGenExplicitAlgorithm(func, _, outInd) and
-        c.getTarget() = func
-      |
-        if outInd = -1 then node.asExpr() = c else node.asExpr() = c.getArgument(outInd)
-      )
-    }
-
-    predicate isSink(DataFlow::Node node) {
-      exists(Call c |
-        isKeyGenOperationWithNoSize(c.getTarget()) and c.getAnArgument() = node.asExpr()
-      )
-    }
-  }
-
-  module KeyGenKeySizeInitToKeyGenFlow = DataFlow::Global<KeyGenKeySizeInitToKeyGenConfig>;
-
-  predicate isEVP_PKEY_CTX_Source(DataFlow::Node node, CryptographicAlgorithm alg) {
-    exists(Call c |
-      alg.configurationSink().(AlgorithmSinkArgument).getSinkCall() = c and
-      (
-        node.asExpr() = c
-        or
-        node.asExpr() = c.getAnArgument()
-        or
-        node.asDefiningArgument() = c.getAnArgument()
-      )
-    ) and
-    (
-      node.asExpr() instanceof Known_EVP_PKEY_CTX_Ptr_Source
-      or
-      node.asDefiningArgument() instanceof Known_EVP_PKEY_CTX_Ptr_Source
-    )
-  }
-
-  predicate isKeyGen_EVP_PKEY_CTX_Sink(DataFlow::Node node, Call c) {
-    isKeyGenOperationWithNoSize(c.getTarget()) and nodeToExpr(node, c.getAnArgument())
-  }
-
-  /**
-   * Trace from EVP_PKEY_CTX* at algorithm sink to keygen,
-   * users can then extrapolatae the matching algorithm from the alg sink to the keygen
-   */
-  module EVP_PKEY_CTX_Ptr_Source_to_KeyGenOperationWithNoSize implements DataFlow::ConfigSig {
-    predicate isSource(DataFlow::Node source) { isEVP_PKEY_CTX_Source(source, _) }
-
-    predicate isSink(DataFlow::Node sink) { isKeyGen_EVP_PKEY_CTX_Sink(sink, _) }
-  }
-
-  module EVP_PKEY_CTX_Ptr_Source_to_KeyGenOperationWithNoSize_Flow =
-    DataFlow::Global<EVP_PKEY_CTX_Ptr_Source_to_KeyGenOperationWithNoSize>;
-
-  /**
-   * UNKNOWN key sizes to general purpose key generation functions (i.e., that take in no key size and assume
-   * is it set on context prior to the call). No path from a key configuration to these operations
-   * means the key size is UNKNOWN, or more precisely the key size is DEFAULT but
-   * the defaults can change with each version of OpenSSL, we simply assume the size is generally UNKNOWN.
-   * ASSUMPTION/TODO: we currently model all known locations where a key size is set explicitly.
-   *                 When a key is set implicitly, this usually means a key generation operation
-   *                 is called where the operation takes in no key size, and no flow to this operation
-   *                 initializes the context with a key size.
-   *                 Currently, without a definitive source (set of sources) to start tracing from, we cannot determine
-   *                 determine if a single path exists that initializes the context with a key size and another that doesn't.
-   *                 Rather than attempt to model all possible sources, we assume that if no path
-   *                 from a key config location reaches a generic key generation operation, then the key size is not set.
-   *                 NOTE: while this is true, it is possible a key size is set in one path, but not in another
-   *                 meaning this approach (and other similar approaches used in this model for UNKNOWN)
-   *                 can produce false negatives.
-   */
-  class OpenSSLDefaultKeyGeneration extends AsymmetricKeyGeneration {
-    OpenSSLDefaultKeyGeneration() {
-      // this is a call to a function matching isKeyGenOperationWithNoSize
-      // and there is no flow from a key configuration source to this call
-      exists(Call c |
-        this = c and
-        isKeyGenOperationWithNoSize(this.getTarget()) and
-        not exists(DataFlow::Node src, DataFlow::Node sink |
-          KeyGenKeySizeInitToKeyGenFlow::flow(src, sink) and
-          nodeToExpr(sink, this.getAnArgument())
-        )
-      )
-    }
-
-    override CryptographicAlgorithm getAlgorithm() {
-      if this.getTarget().getName() in ["DSA_generate_key", "DH_generate_key"]
-      then result = this
-      else
-        // NOTE/ASSUMPTION: EVP_PKEY_keygen, EVP_PKEY_generate assume only other possibilities,
-        //        each take in a CTX as the first arg, need to trace from an alg sink from this CTX param
-        // get every alg sink, get the corresponding call, trace out on any CTX type variable
-        // to the key gen
-        // NOTE: looking for any cryptographic algorithm tracing to the keygen to handle
-        //  any odd cases we aren't awaare of where keygen can be used for other algorithm types
-        exists(DataFlow::Node src, DataFlow::Node sink |
-          EVP_PKEY_CTX_Ptr_Source_to_KeyGenOperationWithNoSize_Flow::flow(src, sink) and
-          isEVP_PKEY_CTX_Source(src, result) and
-          isKeyGen_EVP_PKEY_CTX_Sink(sink, this)
-          // TODO: what if there is no CTX source? then the keygen becomes an UNKNOWN sink
-        )
-    }
-
-    /**
-     * For this class, there is no known configuration source for any algorithm
-     */
-    override Expr getKeyConfigurationSource(CryptographicAlgorithm alg) { none() }
-  }
-}

cpp/ql/lib/experimental/cryptography/utils/OpenSSL/AlgorithmSink.qll

@@ -1,296 +0,0 @@
-/**
- * Predicates/classes for identifying algorithm sinks.
- * An Algorithm Sink is a function that takes an algorithm as an argument.
- * In particular, any function that takes in an algorithm that until the call
- * the algorithm is not definitely known to be an algorithm (e.g., an integer used as an identifier to fetch an algorithm)
- */
-
-//TODO: enforce a hierarchy of AlgorithmSinkArgument, e.g., so I can get all Asymmetric SinkArguments that includes all the strictly RSA etc.
-import cpp
-import experimental.cryptography.utils.OpenSSL.LibraryFunction
-import experimental.cryptography.CryptoAlgorithmNames
-
-predicate isAlgorithmSink(AlgorithmSinkArgument arg, string algType) { arg.algType() = algType }
-
-abstract class AlgorithmSinkArgument extends Expr {
-  AlgorithmSinkArgument() {
-    exists(Call c | c.getAnArgument() = this and openSSLLibraryFunc(c.getTarget()))
-  }
-
-  /**
-   * Gets the function call in which the argument exists
-   */
-  Call getSinkCall() { result.getAnArgument() = this }
-
-  abstract string algType();
-}
-
-// https://www.openssl.org/docs/manmaster/man3/EVP_CIPHER_fetch.html
-predicate cipherAlgorithmSink(string funcName, int argInd) {
-  funcName in ["EVP_get_cipherbyname", "EVP_get_cipherbynid", "EVP_get_cipherbyobj"] and argInd = 0
-  or
-  funcName = "EVP_CIPHER_fetch" and argInd = 1
-}
-
-class CipherAlgorithmSink extends AlgorithmSinkArgument {
-  CipherAlgorithmSink() {
-    exists(Call c, string funcName, int argInd |
-      funcName = c.getTarget().getName() and this = c.getArgument(argInd)
-    |
-      cipherAlgorithmSink(funcName, argInd)
-    )
-  }
-
-  override string algType() { result = getSymmetricEncryptionType() }
-}
-
-// https://www.openssl.org/docs/manmaster/man3/EVP_MAC_fetch
-predicate macAlgorithmSink(string funcName, int argInd) {
-  (funcName = "EVP_MAC_fetch" and argInd = 1)
-}
-
-class MACAlgorithmSink extends AlgorithmSinkArgument {
-  MACAlgorithmSink() {
-    exists(Call c, string funcName, int argInd |
-      funcName = c.getTarget().getName() and this = c.getArgument(argInd)
-    |
-      macAlgorithmSink(funcName, argInd)
-    )
-  }
-
-  override string algType() { result = "TBD" }
-}
-
-// https://www.openssl.org/docs/manmaster/man3/EVP_MD_fetch
-predicate messageDigestAlgorithmSink(string funcName, int argInd) {
-  funcName in ["EVP_get_digestbyname", "EVP_get_digestbynid", "EVP_get_digestbyobj"] and argInd = 0
-  or
-  funcName = "EVP_MD_fetch" and argInd = 1
-}
-
-class MessageDigestAlgorithmSink extends AlgorithmSinkArgument {
-  MessageDigestAlgorithmSink() {
-    exists(Call c, string funcName, int argInd |
-      funcName = c.getTarget().getName() and this = c.getArgument(argInd)
-    |
-      messageDigestAlgorithmSink(funcName, argInd)
-    )
-  }
-
-  override string algType() { result = getHashType() }
-}
-
-// https://www.openssl.org/docs/manmaster/man3/EVP_KEYEXCH_fetch
-// https://www.openssl.org/docs/manmaster/man3/EVP_KEM_fetch
-predicate keyExchangeAlgorithmSink(string funcName, int argInd) {
-  funcName = "EVP_KEYEXCH_fetch" and argInd = 1
-  or
-  funcName = "EVP_KEM_fetch" and argInd = 1
-}
-
-class KeyExchangeAlgorithmSink extends AlgorithmSinkArgument {
-  KeyExchangeAlgorithmSink() {
-    exists(Call c, string funcName, int argInd |
-      funcName = c.getTarget().getName() and this = c.getArgument(argInd)
-    |
-      keyExchangeAlgorithmSink(funcName, argInd)
-    )
-  }
-
-  override string algType() { result = getKeyExchangeType() }
-}
-
-// https://www.openssl.org/docs/manmaster/man3/EVP_KEYMGMT_fetch
-predicate keyManagementAlgorithmSink(string funcName, int argInd) {
-  funcName = "EVP_KEYMGMT_fetch" and argInd = 1
-}
-
-class KeyManagementAlgorithmSink extends AlgorithmSinkArgument {
-  KeyManagementAlgorithmSink() {
-    exists(Call c, string funcName, int argInd |
-      funcName = c.getTarget().getName() and this = c.getArgument(argInd)
-    |
-      keyManagementAlgorithmSink(funcName, argInd)
-    )
-  }
-
-  override string algType() { result = "TBD" }
-}
-
-// https://www.openssl.org/docs/manmaster/man3/EVP_KDF
-predicate keyDerivationAlgorithmSink(string funcName, int argInd) {
-  funcName = "EVP_KDF_fetch" and argInd = 1
-}
-
-class KeyDerivationAlgorithmSink extends AlgorithmSinkArgument {
-  KeyDerivationAlgorithmSink() {
-    exists(Call c, string funcName, int argInd |
-      funcName = c.getTarget().getName() and this = c.getArgument(argInd)
-    |
-      keyDerivationAlgorithmSink(funcName, argInd)
-    )
-  }
-
-  override string algType() { result = getKeyDerivationType() }
-}
-
-// https://www.openssl.org/docs/manmaster/man3/EVP_ASYM_CIPHER_fetch
-// https://www.openssl.org/docs/manmaster/man3/EVP_PKEY_CTX_new_id
-// https://www.openssl.org/docs/manmaster/man3/EVP_PKEY_new_CMAC_key.html
-predicate asymmetricCipherAlgorithmSink(string funcName, int argInd) {
-  funcName = "EVP_ASYM_CIPHER_fetch" and argInd = 1
-  or
-  funcName = "EVP_PKEY_new_CMAC_key" and argInd = 3
-  // NOTE: other cases are handled by AsymmetricAlgorithmSink
-}
-
-class AsymmetricCipherAlgorithmSink extends AlgorithmSinkArgument {
-  AsymmetricCipherAlgorithmSink() {
-    exists(Call c, string funcName, int argInd |
-      funcName = c.getTarget().getName() and this = c.getArgument(argInd)
-    |
-      asymmetricCipherAlgorithmSink(funcName, argInd)
-    )
-  }
-
-  override string algType() { result = "ASYMMETRIC_ENCRYPTION" }
-}
-
-class AsymmetricCipherAlgorithmSink_EVP_PKEY_Q_keygen extends AlgorithmSinkArgument {
-  AsymmetricCipherAlgorithmSink_EVP_PKEY_Q_keygen() {
-    exists(Call c, string funcName |
-      funcName = c.getTarget().getName() and
-      this = c.getArgument(3)
-    |
-      funcName = "EVP_PKEY_Q_keygen" and
-      c.getArgument(3).getType().getUnderlyingType() instanceof IntegralType
-    )
-  }
-
-  override string algType() { result = "ASYMMETRIC_ENCRYPTION" }
-}
-
-// https://www.openssl.org/docs/manmaster/man3/EVP_RAND_fetch
-predicate randomAlgorithmSink(string funcName, int argInd) {
-  funcName = "EVP_RAND_fetch" and argInd = 1
-}
-
-class RandomAlgorithmSink extends AlgorithmSinkArgument {
-  RandomAlgorithmSink() {
-    exists(Call c, string funcName, int argInd |
-      funcName = c.getTarget().getName() and this = c.getArgument(argInd)
-    |
-      randomAlgorithmSink(funcName, argInd)
-    )
-  }
-
-  override string algType() { result = "TBD" }
-}
-
-// https://www.openssl.org/docs/manmaster/man3/EVP_SIGNATURE_fetch
-predicate signatureAlgorithmSink(string funcName, int argInd) {
-  funcName = "EVP_SIGNATURE_fetch" and argInd = 1
-}
-
-class SignatureAlgorithmSink extends AlgorithmSinkArgument {
-  SignatureAlgorithmSink() {
-    exists(Call c, string funcName, int argInd |
-      funcName = c.getTarget().getName() and this = c.getArgument(argInd)
-    |
-      signatureAlgorithmSink(funcName, argInd)
-    )
-  }
-
-  override string algType() { result = getSignatureType() }
-}
-
-// https://www.openssl.org/docs/manmaster/man3/EC_KEY_new_by_curve_name.html
-// https://www.openssl.org/docs/manmaster/man3/EVP_PKEY_CTX_set_ec_paramgen_curve_nid.html
-predicate ellipticCurveAlgorithmSink(string funcName, int argInd) {
-  funcName in ["EC_KEY_new_by_curve_name", "EVP_EC_gen"] and argInd = 0
-  or
-  funcName = "EC_KEY_new_by_curve_name_ex" and argInd = 2
-  or
-  funcName in ["EVP_PKEY_CTX_set_ec_paramgen_curve_nid"] and argInd = 1
-}
-
-class EllipticCurveAlgorithmSink extends AlgorithmSinkArgument {
-  EllipticCurveAlgorithmSink() {
-    exists(Call c, string funcName, int argInd |
-      funcName = c.getTarget().getName() and this = c.getArgument(argInd)
-    |
-      ellipticCurveAlgorithmSink(funcName, argInd)
-    )
-  }
-
-  override string algType() { result = getEllipticCurveType() }
-}
-
-/**
- * Special cased to address the fact that arg index 3 (zero offset based) is the curve name.
- * ASSUMPTION: if the arg ind 3 is a char* assume it is an elliptic curve
- */
-class EllipticCurveAlgorithmSink_EVP_PKEY_Q_keygen extends AlgorithmSinkArgument {
-  EllipticCurveAlgorithmSink_EVP_PKEY_Q_keygen() {
-    exists(Call c, string funcName |
-      funcName = c.getTarget().getName() and
-      this = c.getArgument(3)
-    |
-      funcName = "EVP_PKEY_Q_keygen" and
-      c.getArgument(3).getType().getUnderlyingType() instanceof PointerType and
-      c.getArgument(3).getType().getUnderlyingType().stripType() instanceof CharType
-    )
-  }
-
-  override string algType() { result = getEllipticCurveType() }
-}
-
-// https://www.openssl.org/docs/manmaster/man3/EVP_PKEY_CTX_new_id.html
-// https://www.openssl.org/docs/man1.1.1/man3/EVP_PKEY_new_raw_private_key.html
-// https://www.openssl.org/docs/manmaster/man3/EVP_PKEY_new.html
-// https://www.openssl.org/docs/manmaster/man3/EVP_PKEY_CTX_ctrl.html
-// https://www.openssl.org/docs/manmaster/man3/EVP_PKEY_Q_keygen.html
-// https://www.openssl.org/docs/manmaster/man3/EVP_PKEY_CTX_ctrl.html
-predicate asymmetricAlgorithmSink(string funcName, int argInd) {
-  funcName = "EVP_PKEY_CTX_new_id" and argInd = 0
-  or
-  funcName = "EVP_PKEY_CTX_new_from_name" and argInd = 1
-  or
-  funcName in [
-      "EVP_PKEY_new_raw_private_key", "EVP_PKEY_new_raw_public_key", "EVP_PKEY_new_mac_key"
-    ] and
-  argInd = 0
-  or
-  funcName in ["EVP_PKEY_new_raw_private_key_ex", "EVP_PKEY_new_raw_public_key_ex"] and argInd = 1
-  or
-  // special casing this as arg index 3 must be specified depending on if RSA or ECC, and otherwise not specified for other algs
-  // funcName = "EVP_PKEY_Q_keygen" and argInd = 2
-  funcName in ["EVP_PKEY_CTX_ctrl", "EVP_PKEY_CTX_set_group_name"] and argInd = 1
-  // TODO consider void cases EVP_PKEY_new
-}
-
-class AsymmetricAlgorithmSink extends AlgorithmSinkArgument {
-  AsymmetricAlgorithmSink() {
-    exists(Call c, string funcName, int argInd |
-      funcName = c.getTarget().getName() and this = c.getArgument(argInd)
-    |
-      asymmetricAlgorithmSink(funcName, argInd)
-    )
-  }
-
-  override string algType() { result = getAsymmetricType() }
-}
-
-class AsymmetricAlgorithmSink_EVP_PKEY_Q_keygen extends AlgorithmSinkArgument {
-  AsymmetricAlgorithmSink_EVP_PKEY_Q_keygen() {
-    exists(Call c, string funcName |
-      funcName = c.getTarget().getName() and
-      this = c.getArgument(2)
-    |
-      funcName = "EVP_PKEY_Q_keygen" and
-      not exists(c.getArgument(3))
-    )
-  }
-
-  override string algType() { result = getAsymmetricType() }
-}

cpp/ql/lib/experimental/cryptography/utils/OpenSSL/CryptoFunction.qll

@@ -1,121 +0,0 @@
-import cpp
-import experimental.cryptography.utils.OpenSSL.LibraryFunction
-import experimental.cryptography.CryptoAlgorithmNames
-
-predicate inferredOpenSSLCryptoFunctionCall(Call c, string normalized, string algType) {
-  inferredOpenSSLCryptoFunction(c.getTarget(), normalized, algType)
-}
-
-predicate inferredOpenSSLCryptoFunction(Function f, string normalized, string algType) {
-  isPossibleOpenSSLFunction(f) and
-  normalizeFunctionName(f, algType) = normalized
-}
-
-predicate isOpenSSLCryptoFunction(Function f, string normalized, string algType) {
-  // NOTE: relying on inference as there are thousands of functions for crypto
-  //       enumerating them all and maintaining the list seems problematic.
-  //       For now, we will rely on dynamically inferring algorithms for function names.
-  //       This has been seen to be reasonably efficient and accurate.
-  inferredOpenSSLCryptoFunction(f, normalized, algType)
-}
-
-predicate isOpenSSLCryptoFunctionCall(Call c, string normalized, string algType) {
-  isOpenSSLCryptoFunction(c.getTarget(), normalized, algType)
-}
-
-private string basicNormalizeFunctionName(Function f, string algType) {
-  isPossibleOpenSSLFunction(f) and
-  isKnownAlgorithm(result, algType) and
-  exists(string normStr | normStr = f.getName().toUpperCase().regexpReplaceAll("[-_ ]|/", "") |
-    normStr.matches("%" + result + "%")
-  )
-}
-
-/**
- * Converts a raw OpenSSL algorithm to a normalized algorithm name.
- *
- * If more than one match occurs for a given algorithm type, normalize attempts to find the "max"
- * string (max in terms of string length) e.g., matching AES128 to AES128 and not simply AES.
- *
- * An unknown algorithm is only identified if there exists no known algorithm found for any algorithm type.
- *
- * `f` is the function name to normalize.
- * `algType` is a string representing the classification of the algorithm (see `CryptoAlgorithmNames`)
- */
-private string privateNormalizeFunctionName(Function f, string algType) {
-  isPossibleOpenSSLFunction(f) and
-  result = basicNormalizeFunctionName(f, algType) and
-  not exists(string res2 |
-    result != res2 and
-    res2 = basicNormalizeFunctionName(f, algType) and
-    res2.length() > result.length()
-  )
-}
-
-/**
- * Normalizes a function name to a known algorithm name, similar to `normalizeName`.
- * A function is not, however, allowed to be UNKNOWN. The function either
- * normalizes to a known algorithm name, or the predicate does not hold (no result).
- *
- * The predicate attempts to restrict normalization to what looks like an openssl
- * library by looking for functions only in an openssl path (see `isPossibleOpenSSLFunction`).
- * This may give false postive functions if a directory erronously appears to be openssl;
- * however, we take the stance that if a function
- * exists strongly mapping to a known function name in a directory such as these,
- * regardless of whether its actually a part of openSSL or not, we will analyze it as though it were.
- */
-private string normalizeFunctionName(Function f, string algType) {
-  algType != "UNKNOWN" and
-  isPossibleOpenSSLFunction(f) and
-  result = privateNormalizeFunctionName(f, algType) and
-  // Addressing false positives
-  // For algorithm names less than or equal to 4, we must see the algorithm name
-  // in the original function as upper case (it can't be split between tokens)
-  // One exception found is DES_xcbc_encrypt, this is DESX
-  (
-    (result.length() <= 4 and result != "DESX")
-    implies
-    f.getName().toUpperCase().matches("%" + result + "%")
-  ) and
-  (
-    (result.length() <= 4 and result = "DESX")
-    implies
-    (f.getName().toUpperCase().matches("%DESX%") or f.getName().toUpperCase().matches("%DES_X%"))
-  ) and
-  // (result.length() <= 3 implies (not f.getName().toUpperCase().regexpMatch(".*" + result + "[a-zA-Z0-9].*|.*[a-zA-Z0-9]" + result + ".*")))
-  // and
-  // DES specific false positives
-  (
-    result.matches("DES")
-    implies
-    not f.getName().toUpperCase().regexpMatch(".*DES[a-zA-Z0-9].*|.*[a-zA-Z0-9]DES.*")
-  ) and
-  // ((result.matches("%DES%")) implies not exists(string s | s in ["DESCRIBE", "DESTROY", "DESCRIPTION", "DESCRIPTOR", "NODES"] |
-  //     f.getName().toUpperCase().matches("%" + s + "%"))) and
-  // SEED specific false positives
-  (
-    result.matches("SEED")
-    implies
-    not exists(string s |
-      s in [
-          "SEED_SRC_GENERATE", "RAND", "NEW_SEED", "GEN_SEED", "SEED_GEN", "SET_SEED", "GET_SEED",
-          "GET0_SEED", "RESEED", "SEEDING"
-        ]
-    |
-      f.getName().toUpperCase().matches("%" + s + "%")
-    )
-  ) and
-  // ARIA specific false positives
-  (result.matches("ARIA") implies not f.getName().toUpperCase().matches("%VARIANT%")) and
-  // CTR false positives
-  (result.matches("CTR") implies not f.getName().toUpperCase().matches("%CTRL%")) and
-  // ES false positives (e.g., ES256 from AES256)
-  (result.matches("ES%") implies not f.getName().toUpperCase().matches("%AES%")) and
-  // RSA false positives
-  (result.matches("RSA") implies not f.getName().toUpperCase().matches("%UNIVERSAL%")) and
-  //rsaz functions deemed to be too low level, and can be ignored
-  not f.getLocation().getFile().getBaseName().matches("rsaz_exp.c") and
-  // General False positives
-  // Functions that 'get' do not set an algorithm, and therefore are considered ignorable
-  not f.getName().toLowerCase().matches("%get%")
-}