Merge remote-tracking branch 'origin/dependabot/cargo/ruby/extractor/clap-4.0' into nickrolfe/clap4

Updates the requirements on [clap](https://github.com/clap-rs/clap) to permit the latest version.
- [Release notes](https://github.com/clap-rs/clap/releases)
- [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md)
- [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v3.0.0...v4.0.2)

---
updated-dependencies:
- dependency-name: clap
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/ISSUE_TEMPLATE/lgtm-com---false-positive.md

@@ -0,0 +1,24 @@
+---
+name: LGTM.com - false positive
+about: Tell us about an alert that shouldn't be reported
+title: LGTM.com - false positive
+labels: false-positive
+assignees: ''
+
+---
+
+**Description of the false positive**
+
+<!-- Please explain briefly why you think it shouldn't be included. -->
+
+**URL to the alert on the project page on LGTM.com**
+
+<!--
+1. Open the project on LGTM.com.
+For example, https://lgtm.com/projects/g/pallets/click/.
+2. Switch to the `Alerts` tab. For example, https://lgtm.com/projects/g/pallets/click/alerts/.
+3. Scroll to the alert that you would like to report.
+4. Click on the right most icon `View this alert within the complete file`.
+5. A new browser tab opens. Copy and paste the page URL here.
+For example, https://lgtm.com/projects/g/pallets/click/snapshot/719fb7d8322b0767cdd1e5903ba3eb3233ba8dd5/files/click/_winconsole.py#xa08d213ab3289f87:1.
+-->

.github/ISSUE_TEMPLATE/ql---general.md

@@ -10,5 +10,5 @@ assignees: ''
 **Description of the issue**
 
 <!-- Please explain briefly what is the problem.
-If it is about a GitHub project, please include its URL. -->
+If it is about an LGTM project, please include its URL.-->
 

.github/ISSUE_TEMPLATE/ql--false-positive.md

@@ -1,36 +0,0 @@
----
-name: CodeQL false positive
-about: Report CodeQL alerts that you think should not have been detected (not applicable, not exploitable, etc.)
-title: False positive
-labels: false-positive
-assignees: ''
-
----
-
-**Description of the false positive**
-
-<!-- Please explain briefly why you think it shouldn't be included. -->
-
-**Code samples or links to source code**
-
-<!--
-For open source code: file links with line numbers on GitHub, for example:
-https://github.com/github/codeql/blob/dc440aaee6695deb0d9676b87e06ea984e1b4ae5/javascript/ql/test/query-tests/Security/CWE-078/CommandInjection/exec-sh2.js#L10
-
-For closed source code: (redacted) code samples that illustrate the problem, for example:
-
-```
-function execSh(command, options) {
-    return cp.spawn(getShell(), ["-c", command], options) // <- command line injection
-};
-```
--->
-
-**URL to the alert on GitHub code scanning (optional)**
-
-<!--
-1. Open the project on GitHub.com.
-2. Switch to the `Security` tab.
-3. Browse to the alert that you would like to report.
-4. Copy and paste the page URL here.
--->

.github/actions/cache-query-compilation/action.yml

@@ -1,54 +0,0 @@
-name: Cache query compilation
-description: Caches CodeQL compilation caches - should be run both on PRs and pushes to main.
-
-inputs:
-  key:
-    description: 'The cache key to use - should be unique to the workflow'
-    required: true
-
-outputs:
-  cache-dir:
-    description: "The directory where the cache was stored"
-    value: ${{ steps.fill-compilation-dir.outputs.compdir }}
-
-runs:
-  using: composite
-  steps:
-    # calculate the merge-base with main, in a way that works both on PRs and pushes to main.
-    - name: Calculate merge-base
-      shell: bash
-      if: ${{ github.event_name == 'pull_request' }}
-      env:
-        BASE_BRANCH: ${{ github.base_ref }}
-      run: |
-        MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
-        echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV
-    - name: Restore cache (PR)
-      if: ${{ github.event_name == 'pull_request' }}
-      uses: actions/cache/restore@v3
-      with:
-        path: '**/.cache'
-        key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }}
-        restore-keys: |
-          codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
-          codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-
-          codeql-compile-${{ inputs.key }}-main-
-    - name: Fill cache (only branch push)
-      if: ${{ github.event_name != 'pull_request' }}
-      uses: actions/cache@v3
-      with:
-        path: '**/.cache'
-        key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
-        restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
-          codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
-          codeql-compile-${{ inputs.key }}-main-
-    - name: Fill compilation cache directory
-      id: fill-compilation-dir
-      shell: bash
-      run: |
-        # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-        node $GITHUB_WORKSPACE/.github/actions/cache-query-compilation/move-caches.js ${COMBINED_CACHE_DIR}
-
-        echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
-      env:
-        COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir

.github/actions/cache-query-compilation/move-caches.js

@@ -1,75 +0,0 @@
-// # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-// mkdir -p ${COMBINED_CACHE_DIR}
-// rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-// # copy the contents of the .cache folders into the combined cache folder.
-// cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-// # clean up the .cache folders
-// rm -rf **/.cache/*
-
-const fs = require("fs");
-const path = require("path");
-
-// the first argv is the cache folder to create.
-const COMBINED_CACHE_DIR = process.argv[2];
-
-function* walkCaches(dir) {
-  const files = fs.readdirSync(dir, { withFileTypes: true });
-  for (const file of files) {
-    if (file.isDirectory()) {
-      const filePath = path.join(dir, file.name);
-      yield* walkCaches(filePath);
-      if (file.name === ".cache") {
-        yield filePath;
-      }
-    }
-  }
-}
-
-async function copyDir(src, dest) {
-  for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
-    const srcPath = path.join(src, file.name);
-    const destPath = path.join(dest, file.name);
-    if (file.isDirectory()) {
-      if (!fs.existsSync(destPath)) {
-        fs.mkdirSync(destPath);
-      }
-      await copyDir(srcPath, destPath);
-    } else {
-      await fs.promises.copyFile(srcPath, destPath);
-    }
-  }
-}
-
-async function main() {
-  const cacheDirs = [...walkCaches(".")];
-
-  for (const dir of cacheDirs) {
-    console.log(`Found .cache dir at ${dir}`);
-  }
-
-  // mkdir -p ${COMBINED_CACHE_DIR}
-  fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
-
-  // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-  await Promise.all(
-    cacheDirs.map((cacheDir) =>
-      (async function () {
-        await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
-        await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
-      })()
-    )
-  );
-
-  // # copy the contents of the .cache folders into the combined cache folder.
-  // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-  await Promise.all(
-    cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
-  );
-
-  // # clean up the .cache folders
-  // rm -rf **/.cache/*
-  await Promise.all(
-    cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
-  );
-}
-main();

.github/actions/fetch-codeql/action.yml

@@ -1,24 +1,14 @@
 name: Fetch CodeQL
 description: Fetches the latest version of CodeQL
-
-inputs:
-  channel:
-    description: 'The CodeQL channel to use'
-    required: false
-    default: 'nightly'
-
 runs:
   using: composite
   steps:
     - name: Fetch CodeQL
       shell: bash
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-        CHANNEL: ${{ inputs.channel }}
       run: |
         gh extension install github/gh-codeql
-        gh codeql set-channel "$CHANNEL"
+        gh codeql set-channel nightly
         gh codeql version
-        printf "CODEQL_FETCHED_CODEQL_PATH=" >> "${GITHUB_ENV}"
-        gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_ENV}"
         gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_PATH}"
+      env:
+        GITHUB_TOKEN: ${{ github.token }}

.github/actions/find-latest-bundle/action.yml

@@ -1,26 +0,0 @@
-name: Find Latest CodeQL Bundle
-description: Finds the URL of the latest released version of the CodeQL bundle.
-outputs:
-  url:
-    description: The download URL of the latest CodeQL bundle release
-    value: ${{ steps.find-latest.outputs.url }}
-runs:
-  using: composite
-  steps:
-    - name: Find Latest Release
-      id: find-latest
-      shell: pwsh
-      run: |
-        $Latest = gh release list --repo github/codeql-action --exclude-drafts --limit 1000 |
-          ForEach-Object { $C = $_ -split "`t"; return @{ type = $C[1]; tag = $C[2]; } } |
-          Where-Object { $_.type -eq 'Latest' }
-
-        $Tag = $Latest.tag
-        if ($Tag -eq '') {
-          throw 'Failed to find latest bundle release.'
-        }
-
-        Write-Output "Latest bundle tag is '${Tag}'."
-        "url=https://github.com/github/codeql-action/releases/download/${Tag}/codeql-bundle-linux64.tar.gz" >> $env:GITHUB_OUTPUT
-      env:
-        GITHUB_TOKEN: ${{ github.token }}

.github/actions/os-version/action.yml

@@ -1,32 +0,0 @@
-name: OS Version
-description: Get OS version.
-
-outputs:
-  version:
-    description: "OS version"
-    value: ${{ steps.version.outputs.version }}
-
-runs:
-  using: composite
-  steps:
-    - if: runner.os == 'Linux'
-      shell: bash
-      run: |
-        . /etc/os-release
-        echo "VERSION=${NAME} ${VERSION}" >> $GITHUB_ENV
-    - if: runner.os == 'Windows'
-      shell: powershell
-      run: |
-        $objects = systeminfo.exe /FO CSV | ConvertFrom-Csv
-        "VERSION=$($objects.'OS Name') $($objects.'OS Version')" >> $env:GITHUB_ENV
-    - if: runner.os == 'macOS'
-      shell: bash
-      run: |
-        echo "VERSION=$(sw_vers -productName) $(sw_vers -productVersion)" >> $GITHUB_ENV
-    - name: Emit OS version
-      id: version
-      shell: bash
-      run: |
-        echo "$VERSION"
-        echo "version=${VERSION}" >> $GITHUB_OUTPUT
-

.github/labeler.yml

@@ -43,14 +43,3 @@ documentation:
 "QL-for-QL":
   - ql/**/*
   - .github/workflows/ql-for-ql*
-
-# Since these are all shared files that need to be synced, just pick _one_ copy of each.
-"DataFlow Library":
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"
-
-"ATM":
-  - javascript/ql/experimental/adaptivethreatmodeling/**/*

.github/workflows/atm-check-query-suite.yml

@@ -1,102 +0,0 @@
-name: "ATM - Check query suite"
-
-env:
-  QUERY_PACK: javascript/ql/experimental/adaptivethreatmodeling/src
-  QUERY_SUITE: codeql-suites/javascript-atm-code-scanning.qls
-
-on:
-  pull_request:
-    paths:
-      - ".github/workflows/atm-check-query-suite.yml"
-      - "javascript/ql/experimental/adaptivethreatmodeling/**"
-  workflow_dispatch:
-
-jobs:
-  atm-check-query-suite:
-    runs-on: ubuntu-latest-xl
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Setup CodeQL
-        uses: ./.github/actions/fetch-codeql
-        with:
-          channel: release
-
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: atm-suite
-
-      - name: Install ATM model
-        run: |
-          set -exu
-
-          # Install dependencies of ATM query pack, i.e. the ATM model
-          codeql pack install "${QUERY_PACK}"
-
-          # Retrieve model checksum
-          model_checksum=$(codeql resolve extensions "${QUERY_PACK}/${QUERY_SUITE}" | jq -r '.models[0].checksum')
-
-          # Trust the model so that we can use it in the ATM boosted queries
-          mkdir -p "$HOME/.config/codeql"
-          echo "--insecurely-execute-ml-model-checksums ${model_checksum}" >> "$HOME/.config/codeql/config"
-
-      - name: Create test DB
-        run: |
-          DB_PATH="${RUNNER_TEMP}/db"
-          echo "DB_PATH=${DB_PATH}" >> "${GITHUB_ENV}"
-
-          codeql database create "${DB_PATH}" --source-root config/atm --language javascript 
-
-      - name: Run ATM query suite
-        run: |
-          SARIF_PATH="${RUNNER_TEMP}/sarif.json"
-          echo "SARIF_PATH=${SARIF_PATH}" >> "${GITHUB_ENV}"
-
-          codeql database analyze \
-            --threads=0 \
-            --ram 50000 \
-            --format sarif-latest \
-            --output "${SARIF_PATH}" \
-            --sarif-group-rules-by-pack \
-            -vv \
-            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
-            -- \
-            "${DB_PATH}" \
-            "${QUERY_PACK}/${QUERY_SUITE}"
-
-      - name: Upload SARIF
-        uses: actions/upload-artifact@v3
-        with:
-          name: javascript-ml-powered-queries.sarif
-          path: "${{ env.SARIF_PATH }}"
-          retention-days: 5
-
-      - name: Check results
-        run: |
-          # We should run at least the ML-powered queries in `expected_rules`.
-          expected_rules="js/ml-powered/nosql-injection js/ml-powered/path-injection js/ml-powered/sql-injection js/ml-powered/xss"
-
-          for rule in ${expected_rules}; do
-            found_rule=$(jq --arg rule "${rule}" '[.runs[0].tool.extensions[].rules | select(. != null) |
-              flatten | .[].id] | any(. == $rule)' "${SARIF_PATH}")
-            if [[ "${found_rule}" != "true" ]]; then
-              echo "Expected SARIF output to contain rule '${rule}', but found no such rule."
-              exit 1
-            else
-              echo "Found rule '${rule}'."
-            fi
-          done
-
-          # We should have at least one alert from an ML-powered query.
-          num_alerts=$(jq '[.runs[0].results[] |
-            select(.properties.score != null and (.rule.id | startswith("js/ml-powered/")))] | length' \
-            "${SARIF_PATH}")
-          if [[ "${num_alerts}" -eq 0 ]]; then
-            echo "Expected to find at least one alert from an ML-powered query but found ${num_alerts}."
-            exit 1
-          else
-            echo "Found ${num_alerts} alerts from ML-powered queries.";
-          fi

.github/workflows/atm-model-integration-tests.yml

@@ -1,12 +0,0 @@
-name: ATM Model Integration Tests
-
-on:
-  workflow_dispatch:
-
-jobs:
-  hello-world:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: foo
-        run: echo "Hello world"

.github/workflows/check-change-note.yml

@@ -26,9 +26,3 @@ jobs:
         run: |
           gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' |
           grep true -c
-      - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md' or 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text.
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$"))' |
-          grep true -c

.github/workflows/check-query-ids.yml

@@ -1,21 +0,0 @@
-name: Check query IDs
-
-on:
-  pull_request:
-    paths:
-      - "**/src/**/*.ql"
-      - misc/scripts/check-query-ids.py
-      - .github/workflows/check-query-ids.yml
-    branches:
-      - main
-      - "rc/*"
-  workflow_dispatch:
-
-jobs:
-  check:
-    name: Check query IDs
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check for duplicate query IDs
-        run: python3 misc/scripts/check-query-ids.py

.github/workflows/close-stale.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/stale@v7
+    - uses: actions/stale@v6
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Comment or remove the `Stale` label in order to avoid having this issue closed in 7 days.'

.github/workflows/compile-queries.yml

@@ -1,37 +0,0 @@
-name: "Compile all queries using the latest stable CodeQL CLI"
-
-on:
-  push:
-    branches:  # makes sure the cache gets populated - running on the branches people tend to merge into.
-      - main
-      - "rc/*"
-      - "codeql-cli-*"
-  pull_request:
-
-jobs:
-  compile-queries:
-    runs-on: ubuntu-latest-xl
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup CodeQL
-        uses: ./.github/actions/fetch-codeql
-        with:
-          channel: 'release'
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: all-queries
-      - name: check formatting
-        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
-      - name: compile queries - check-only
-        # run with --check-only if running in a PR (github.sha != main)
-        if : ${{ github.event_name == 'pull_request' }}
-        shell: bash
-        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-      - name: compile queries - full
-        # do full compile if running on main - this populates the cache
-        if : ${{ github.event_name != 'pull_request' }}
-        shell: bash
-        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/csharp-qltest.yml

@@ -1,86 +0,0 @@
-name: "C#: Run QL Tests"
-
-on:
-  push:
-    paths:
-      - "csharp/**"
-      - "shared/**"
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "csharp/**"
-      - "shared/**"
-      - .github/workflows/csharp-qltest.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
-
-defaults:
-  run:
-    working-directory: csharp
-
-jobs:
-  qlupgrade:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - name: Check DB upgrade scripts
-        run: |
-          echo >empty.trap
-          codeql dataset import -S ql/lib/upgrades/initial/semmlecode.csharp.dbscheme testdb empty.trap
-          codeql dataset upgrade testdb --additional-packs ql/lib
-          diff -q testdb/semmlecode.csharp.dbscheme ql/lib/semmlecode.csharp.dbscheme
-      - name: Check DB downgrade scripts
-        run: |
-          echo >empty.trap
-          rm -rf testdb; codeql dataset import -S ql/lib/semmlecode.csharp.dbscheme testdb empty.trap
-          codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
-           --dbscheme=ql/lib/semmlecode.csharp.dbscheme --target-dbscheme=downgrades/initial/semmlecode.csharp.dbscheme |
-           xargs codeql execute upgrades testdb
-          diff -q testdb/semmlecode.csharp.dbscheme downgrades/initial/semmlecode.csharp.dbscheme
-  qltest:
-    runs-on: ubuntu-latest-xl
-    strategy:
-      fail-fast: false
-      matrix:
-        slice: ["1/2", "2/2"]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - uses: ./csharp/actions/create-extractor-pack
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with:
-          key: csharp-qltest-${{ matrix.slice }}
-      - name: Run QL tests
-        run: |
-          CODEQL_PATH=$(gh codeql version --format=json | jq -r .unpackedLocation)
-          # The legacy ASP extractor is not in this repo, so take the one from the nightly build
-          mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
-          # Safe guard against using the bundled extractor
-          rm -rf "$CODEQL_PATH/csharp"
-          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-  unit-tests:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup dotnet
-        uses: actions/setup-dotnet@v3
-        with:
-          dotnet-version: 6.0.202
-      - name: Extractor unit tests
-        run: |
-          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Util.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Extraction.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"

.github/workflows/go-tests-other-os.yml

@@ -1,80 +0,0 @@
-name: "Go: Run Tests - Other OS"
-on:
-  pull_request:
-    paths:
-      - "go/**"
-      - "!go/ql/**" # don't run other-os if only ql/ files changed
-      - .github/workflows/go-tests-other-os.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-jobs:
-  test-mac:
-    name: Test MacOS
-    runs-on: macos-latest
-    steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.19
-        id: go
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up CodeQL CLI
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Enable problem matchers in repository
-        shell: bash
-        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-      - name: Build
-        run: |
-          cd go
-          make
-
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with:
-          key: go-qltest
-      - name: Test
-        run: |
-          cd go
-          make test cache="${{ steps.query-cache.outputs.cache-dir }}"
-
-  test-win:
-    name: Test Windows
-    runs-on: windows-latest-xl
-    steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.19
-        id: go
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up CodeQL CLI
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Enable problem matchers in repository
-        shell: bash
-        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-      - name: Build
-        run: |
-          cd go
-          make
-
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with:
-          key: go-qltest
-
-      - name: Test
-        run: |
-          cd go
-          make test cache="${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/go-tests.yml

@@ -1,24 +1,15 @@
 name: "Go: Run Tests"
 on:
-  push:
-    paths:
-      - "go/**"
-      - .github/workflows/go-tests.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
   pull_request:
     paths:
       - "go/**"
       - .github/workflows/go-tests.yml
-      - .github/actions/**
+      - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
 jobs:
   test-linux:
     name: Test Linux (Ubuntu)
-    runs-on: ubuntu-latest-xl
+    runs-on: ubuntu-latest
     steps:
       - name: Set up Go 1.19
         uses: actions/setup-go@v3
@@ -41,7 +32,7 @@ jobs:
           cd go
           make
 
-      - name: Check that all Go code is autoformatted
+      - name: Check that all QL and Go code is autoformatted
         run: |
           cd go
           make check-formatting
@@ -57,13 +48,67 @@ jobs:
           name: qhelp-markdown
           path: go/qhelp-out/**/*.md
 
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
+      - name: Test
+        run: |
+          cd go
+          make test
+
+  test-mac:
+    name: Test MacOS
+    runs-on: macos-latest
+    steps:
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
         with:
-          key: go-qltest
+          go-version: 1.19
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
 
       - name: Test
         run: |
           cd go
-          make test cache="${{ steps.query-cache.outputs.cache-dir }}"
+          make test
+
+  test-win:
+    name: Test Windows
+    runs-on: windows-2019
+    steps:
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Test
+        run: |
+          cd go
+          make test

.github/workflows/js-ml-tests.yml

@@ -23,9 +23,22 @@ defaults:
     working-directory: javascript/ql/experimental/adaptivethreatmodeling
 
 jobs:
-  qltest:
-    name: Test QL
-    runs-on: ubuntu-latest-xl
+  qlformat:
+    name: Check QL formatting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Check QL formatting
+        run: |
+          find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
+            xargs -0 codeql query format --check-only
+
+  qlcompile:
+    name: Check QL compilation
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
@@ -33,33 +46,36 @@ jobs:
 
       - name: Install pack dependencies
         run: |
-          for pack in modelbuilding src test; do
+          for pack in modelbuilding src; do
             codeql pack install --mode verify -- "${pack}"
           done
-      
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: js-ml-test
 
       - name: Check QL compilation
         run: |
           codeql query compile \
             --check-only \
-            --ram 50000 \
+            --ram 5120 \
             --additional-packs "${{ github.workspace }}" \
             --threads=0 \
-            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
             -- \
             lib modelbuilding src
 
+  qltest:
+    name: Run QL tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Install pack dependencies
+        run: codeql pack install -- test
+
       - name: Run QL tests
         run: |
           codeql test run \
             --threads=0 \
-            --ram 50000 \
+            --ram 5120 \
             --additional-packs "${{ github.workspace }}" \
-            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
             -- \
-            test
+            test

.github/workflows/mad_modelDiff.yml

@@ -11,7 +11,7 @@ on:
     branches:
       - main
     paths:
-      - "java/ql/src/utils/modelgenerator/**/*.*"
+      - "java/ql/src/utils/model-generator/**/*.*"
       - ".github/workflows/mad_modelDiff.yml"
 
 permissions:
@@ -40,12 +40,12 @@ jobs:
       - name: Download database
         env:
           SLUG: ${{ matrix.slug }}
-          GH_TOKEN: ${{ github.token }}
         run: |
           set -x
           mkdir lib-dbs
           SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
-          gh api -H "Accept: application/zip" "/repos/${SLUG}/code-scanning/codeql/databases/java" > "$SHORTNAME.zip"
+          projectId=`curl -s https://lgtm.com/api/v1.0/projects/g/${SLUG} | jq .id`
+          curl -L "https://lgtm.com/api/v1.0/snapshots/$projectId/java" -o "$SHORTNAME.zip"
           unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
           mkdir "lib-dbs/$SHORTNAME/"
           mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/"
@@ -61,8 +61,8 @@ jobs:
             DATABASE=$2
             cd codeql-$QL_VARIANT
             SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE ${SHORTNAME}.temp.model.yml
-            mv java/ql/lib/ext/generated/${SHORTNAME}.temp.model.yml $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.model.yml
+            python java/ql/src/utils/model-generator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $MODELS/${SHORTNAME}.qll
+            mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
             cd ..
           }
 
@@ -85,21 +85,19 @@ jobs:
           set -x
           MODELS=`pwd`/tmp-models
           ls -1 tmp-models/
-          for m in $MODELS/*_main.model.yml ; do
+          for m in $MODELS/*_main.qll ; do
             t="${m/main/"pr"}"
             basename=`basename $m`
-            name="diff_${basename/_main.model.yml/""}"
+            name="diff_${basename/_main.qll/""}"
             (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
           done
       - uses: actions/upload-artifact@v3
         with:
           name: models
-          path: tmp-models/*.model.yml
+          path: tmp-models/*.qll
           retention-days: 20
       - uses: actions/upload-artifact@v3
         with:
           name: diffs
           path: tmp-models/*.html
-          # An html file is only produced if the generated models differ.
-          if-no-files-found: ignore
           retention-days: 20

.github/workflows/mad_regenerate-models.yml

@@ -50,10 +50,10 @@ jobs:
           SLUG: ${{ matrix.slug }}
         run: |
           SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
-          java/ql/src/utils/modelgenerator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
+          java/ql/src/utils/model-generator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
       - name: Stage changes
         run: |
-          find java -name "*.model.yml" -print0 | xargs -0 git add
+          find java -name "*.qll" -print0 | xargs -0 git add
           git status
           git diff --cached > models.patch
       - uses: actions/upload-artifact@v3

.github/workflows/qhelp-pr-preview.yml

@@ -27,7 +27,7 @@ on:
       - main
       - "rc/*"
     paths:
-      - "**/*.qhelp"
+      - "ruby/**/*.qhelp"
 
 jobs:
   qhelp:
@@ -52,7 +52,7 @@ jobs:
         id: changes
         run: |
           (git diff -z --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep -z '.qhelp$' | grep -z -v '.inc.qhelp';
-           git diff -z --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep -z '.inc.qhelp$' | xargs --null -rn1 basename -z | xargs --null -rn1 git grep -z -l) |
+           git diff -z --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep -z '.inc.qhelp$' | xargs --null -rn1 basename | xargs --null -rn1 git grep -z -l) |
            grep -z '.qhelp$' | grep -z -v '^-' | sort -z -u > "${RUNNER_TEMP}/paths.txt"
 
       - name: QHelp preview

.github/workflows/ql-for-ql-build.yml

@@ -22,30 +22,24 @@ jobs:
     steps:
       ### Build the queries ###
       - uses: actions/checkout@v3
-      - name: Find latest bundle
-        id: find-latest-bundle
-        uses: ./.github/actions/find-latest-bundle
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: javascript # does not matter
-          tools: ${{ steps.find-latest-bundle.outputs.url }}
       - name: Get CodeQL version
         id: get-codeql-version
         run: |
-          echo "version=$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)" >> $GITHUB_OUTPUT
+          echo "::set-output name=version::$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)"
         shell: bash
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - uses: ./.github/actions/os-version
-        id: os_version
       - name: Cache entire pack
         id: cache-pack
         uses: actions/cache@v3
         with:
           path: ${{ runner.temp }}/pack
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-pack-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
+          key: ${{ runner.os }}-pack-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
       - name: Cache queries
         if: steps.cache-pack.outputs.cache-hit != 'true'
         id: cache-queries
@@ -79,7 +73,7 @@ jobs:
             ql/target/release/ql-autobuilder.exe
             ql/target/release/ql-extractor
             ql/target/release/ql-extractor.exe
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
+          key: ${{ runner.os }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
       - name: Cache cargo
         if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         uses: actions/cache@v3
@@ -88,7 +82,7 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
+          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Check formatting
         if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         run: cd ql; cargo fmt --all -- --check
@@ -139,20 +133,19 @@ jobs:
         env:
           CONF: ./ql-for-ql-config.yml
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: ql
           db-location: ${{ runner.temp }}/db
           config-file: ./ql-for-ql-config.yml
-          tools: ${{ steps.find-latest-bundle.outputs.url }}
-      - name: Move pack queries
+      - name: Move pack cache
         run: |
-          cp -r ${PACK}/queries ql/ql/src
+          cp -r ${PACK}/.cache ql/ql/src/.cache
         env:
           PACK: ${{ runner.temp }}/pack
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@45955cb1830b640e2c1603ad72ad542a49d47b96
+        uses: github/codeql-action/analyze@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           category: "ql-for-ql"
       - name: Copy sarif file to CWD
@@ -174,4 +167,4 @@ jobs:
         with:
           name: ql-for-ql-langs
           path: split-sarif
-          retention-days: 1
+          retention-days: 1

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -25,18 +25,16 @@ jobs:
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: javascript # does not matter
-      - uses: ./.github/actions/os-version
-        id: os_version
       - uses: actions/cache@v3
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
+          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Build Extractor
         run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./scripts/create-extractor-pack.sh
         env:

.github/workflows/ql-for-ql-tests.yml

@@ -22,84 +22,33 @@ jobs:
       - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: javascript # does not matter
-      - uses: ./.github/actions/os-version
-        id: os_version
       - uses: actions/cache@v3
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/rust-toolchain.toml', 'ql/**/Cargo.lock') }}
+          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Build extractor
         run: |
           cd ql;
           codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
           env "PATH=$PATH:$codeqlpath" ./scripts/create-extractor-pack.sh
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: ql-for-ql-tests
       - name: Run QL tests
         run: |
-          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
+          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-
-  other-os: 
-    strategy:
-      matrix:
-        os: [macos-latest, windows-latest]
-    needs: [qltest]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install GNU tar 
-        if: runner.os == 'macOS'
+      - name: Check QL formatting
         run: |
-          brew install gnu-tar
-          echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
-        with:
-          languages: javascript # does not matter
-      - uses: ./.github/actions/os-version
-        id: os_version
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/rust-toolchain.toml', 'ql/**/Cargo.lock') }}
-      - name: Build extractor
-        if: runner.os != 'Windows'
-        run: |
-          cd ql;
-          codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
-          env "PATH=$PATH:$codeqlpath" ./scripts/create-extractor-pack.sh
-      - name: Build extractor (Windows)
-        if: runner.os == 'Windows'
-        shell: pwsh
-        run: |
-          cd ql;
-          $Env:PATH += ";$(dirname ${{ steps.find-codeql.outputs.codeql-path }})"
-          pwsh ./scripts/create-extractor-pack.ps1
-      - name: Run a single QL tests - Unix
-        if: runner.os != 'Windows'
-        run: |
-          "${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
+          find ql/ql/src "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Run a single QL tests - Windows
-        if: runner.os == 'Windows'
-        shell: pwsh
+      - name: Check QL compilation
         run: |
-          $Env:PATH += ";$(dirname ${{ steps.find-codeql.outputs.codeql-path }})"
-          codeql test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
-      
+          "${CODEQL}" query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}/ql/extractor-pack" "ql/ql/src" "ql/ql/examples"
+        env:
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}

.github/workflows/ruby-build.yml

@@ -48,41 +48,23 @@ jobs:
         run: |
           brew install gnu-tar
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
-      - uses: ./.github/actions/os-version
-        id: os_version
-      - name: Cache entire extractor
-        uses: actions/cache@v3
-        id: cache-extractor
-        with:
-          path: |
-            ruby/target/release/ruby-autobuilder
-            ruby/target/release/ruby-autobuilder.exe
-            ruby/target/release/ruby-extractor
-            ruby/target/release/ruby-extractor.exe
-            ruby/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}--${{ hashFiles('ruby/**/*.rs') }}
       - uses: actions/cache@v3
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
             ruby/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-rust-cargo-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}
+          key: ${{ runner.os }}-ruby-rust-cargo-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}
       - name: Check formatting
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cargo fmt --all -- --check
       - name: Build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cargo build --verbose
       - name: Run tests
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cargo test --verbose
       - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cargo build --release
       - name: Generate dbscheme
-        if: ${{ matrix.os == 'ubuntu-latest' && steps.cache-extractor.outputs.cache-hit != 'true'}}
+        if: ${{ matrix.os == 'ubuntu-latest' }}
         run: target/release/ruby-generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v3
         if: ${{ matrix.os == 'ubuntu-latest' }}
@@ -104,25 +86,19 @@ jobs:
             ruby/target/release/ruby-extractor.exe
           retention-days: 1
   compile-queries:
-    runs-on: ubuntu-latest-xl
+    runs-on: ubuntu-latest
+    env:
+      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
     steps:
       - uses: actions/checkout@v3
       - name: Fetch CodeQL
         uses: ./.github/actions/fetch-codeql
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: ruby-build
       - name: Build Query Pack
         run: |
-          rm -rf target/packs
-          codeql pack create ../misc/suite-helpers --output target/packs
-          codeql pack create ../shared/regex --output target/packs
           codeql pack create ../shared/ssa --output target/packs
-          codeql pack create ../shared/tutorial --output target/packs
           codeql pack create ql/lib --output target/packs
-          codeql pack create -j0 ql/src --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql pack install ql/src
+          codeql pack create ql/src --output target/packs
           PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
           codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
           (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
@@ -205,6 +181,11 @@ jobs:
       - name: Fetch CodeQL
         uses: ./.github/actions/fetch-codeql
 
+      - uses: actions/checkout@v3
+        with:
+          repository: Shopify/example-ruby-app
+          ref: 67a0decc5eb550f3a9228eda53925c3afd40dfe9
+
       - name: Download Ruby bundle
         uses: actions/download-artifact@v3
         with:
@@ -213,15 +194,26 @@ jobs:
       - name: Unzip Ruby bundle
         shell: bash
         run: unzip -q -d "${{ runner.temp }}/ruby-bundle" "${{ runner.temp }}/codeql-ruby-bundle.zip"
-
+      - name: Prepare test files
+        shell: bash
+        run: |
+          echo "import codeql.ruby.AST select count(File f)" > "test.ql"
+          echo "| 4 |" > "test.expected"
+          echo 'name: sample-tests
+          version: 0.0.0
+          dependencies:
+            codeql/ruby-all: 0.0.1
+          extractor: ruby
+          tests: .
+          ' > qlpack.yml
       - name: Run QL test
         shell: bash
         run: |
-          codeql test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" ruby/ql/test/library-tests/ast/constants/
+          codeql test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
       - name: Create database
         shell: bash
         run: |
-          codeql database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root ruby/ql/test/library-tests/ast/constants/ ../database
+          codeql database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
       - name: Analyze database
         shell: bash
         run: |

.github/workflows/ruby-qltest.yml

@@ -4,7 +4,7 @@ on:
   push:
     paths:
       - "ruby/**"
-      - .github/workflows/ruby-build.yml
+      - .github/workflows/ruby-qltest.yml
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
     branches:
@@ -28,6 +28,23 @@ defaults:
     working-directory: ruby
 
 jobs:
+  qlformat:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - name: Check QL formatting
+        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
+  qlcompile:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - name: Check QL compilation
+        run: |
+          codeql query compile --check-only --threads=0 --ram 5000 --warnings=error "ql/src" "ql/examples"
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
   qlupgrade:
     runs-on: ubuntu-latest
     steps:
@@ -48,20 +65,17 @@ jobs:
            xargs codeql execute upgrades testdb
           diff -q testdb/ruby.dbscheme downgrades/initial/ruby.dbscheme
   qltest:
-    runs-on: ubuntu-latest-xl
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
+      matrix:
+        slice: ["1/2", "2/2"]
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - uses: ./ruby/actions/create-extractor-pack
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: ruby-qltest
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 5000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/swift-codegen.yml

@@ -0,0 +1,39 @@
+name: "Swift: Check code generation"
+
+on:
+  pull_request:
+    paths:
+      - "swift/**"
+      - "misc/bazel/**"
+      - "*.bazel*"
+      - .github/workflows/swift-codegen.yml
+      - .github/actions/fetch-codeql/action.yml
+    branches:
+      - main
+
+jobs:
+  codegen:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v3
+      - uses: pre-commit/action@v3.0.0
+        name: Check that python code is properly formatted
+        with:
+          extra_args: autopep8 --all-files
+      - name: Run unit tests
+        run: |
+          bazel test //swift/codegen/test --test_output=errors
+      - uses: pre-commit/action@v3.0.0
+        name: Check that QL generated code was checked in
+        with:
+          extra_args: swift-codegen --all-files
+      - name: Generate C++ files
+        run: |
+          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-cpp-files
+      - uses: actions/upload-artifact@v3
+        with:
+          name: swift-generated-cpp-files
+          path: swift-generated-cpp-files/**

.github/workflows/swift-integration-tests.yml

@@ -0,0 +1,45 @@
+name: "Swift: Run Integration Tests"
+
+on:
+  pull_request:
+    paths:
+      - "swift/**"
+      - "misc/bazel/**"
+      - "*.bazel*"
+      - .github/workflows/swift-integration-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+defaults:
+  run:
+    working-directory: swift
+
+jobs:
+  integration-tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-20.04
+#          - macos-latest  TODO
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v3
+      - name: Build Swift extractor
+        run: |
+          bazel run //swift:create-extractor-pack
+      - name: Get Swift version
+        id: get_swift_version
+        run: |
+          VERSION=$(bazel run //swift/extractor -- --version | sed -ne 's/.*version \(\S*\).*/\1/p')
+          echo "::set-output name=version::$VERSION"
+      - uses: swift-actions/setup-swift@v1
+        with:
+          swift-version: "${{steps.get_swift_version.outputs.version}}"
+      - name: Run integration tests
+        run: |
+          python integration-tests/runner.py

.github/workflows/swift-qltest.yml

@@ -0,0 +1,43 @@
+name: "Swift: Run QL Tests"
+
+on:
+  pull_request:
+    paths:
+      - "swift/**"
+      - "misc/bazel/**"
+      - "*.bazel*"
+      - .github/workflows/swift-qltest.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+defaults:
+  run:
+    working-directory: swift
+
+jobs:
+  qlformat:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - name: Check QL formatting
+        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
+  qltest:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os : [ubuntu-20.04, macos-latest]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - uses: bazelbuild/setup-bazelisk@v2
+      - name: Build Swift extractor
+        run: |
+          bazel run //swift:create-extractor-pack
+      - name: Run QL tests
+        run: |
+          codeql test run --threads=0 --ram 5000 --search-path "${{ github.workspace }}/swift/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition ql/test
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/swift.yml

@@ -1,103 +0,0 @@
-name: "Swift"
-
-on:
-  pull_request:
-    paths:
-      - "swift/**"
-      - "misc/bazel/**"
-      - "*.bazel*"
-      - .github/workflows/swift.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-      - .pre-commit-config.yaml
-      - "!**/*.md"
-      - "!**/*.qhelp"
-    branches:
-      - main
-      - rc/*
-  push:
-    paths:
-      - "swift/**"
-      - "misc/bazel/**"
-      - "*.bazel*"
-      - .github/workflows/swift.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-      - "!**/*.md"
-      - "!**/*.qhelp"
-    branches:
-      - main
-      - rc/*
-
-jobs:
-  # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks
-  # without waiting for the macOS build
-  build-and-test-macos:
-    runs-on: macos-12-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/build-and-test
-  build-and-test-linux:
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/build-and-test
-  qltests-linux:
-    needs: build-and-test-linux
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/run-ql-tests
-  qltests-macos:
-    if : ${{ github.event_name == 'pull_request' }}
-    needs: build-and-test-macos
-    runs-on: macos-12-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/run-ql-tests
-  integration-tests-linux:
-    needs: build-and-test-linux
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/run-integration-tests
-  integration-tests-macos:
-    if : ${{ github.event_name == 'pull_request' }}
-    needs: build-and-test-macos
-    runs-on: macos-12-xl
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/run-integration-tests
-  codegen:
-    if : ${{ github.event_name == 'pull_request' }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: bazelbuild/setup-bazelisk@v2
-      - uses: actions/setup-python@v4
-        with:
-          python-version-file: 'swift/.python-version'
-      - uses: pre-commit/action@v3.0.0
-        name: Check that python code is properly formatted
-        with:
-          extra_args: autopep8 --all-files
-      - uses: ./.github/actions/fetch-codeql
-      - uses: pre-commit/action@v3.0.0
-        name: Check that QL generated code was checked in
-        with:
-          extra_args: swift-codegen --all-files
-      - name: Generate C++ files
-        run: |
-          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/generated-cpp-files
-      - uses: actions/upload-artifact@v3
-        with:
-          name: swift-generated-cpp-files
-          path: generated-cpp-files/**
-  database-upgrade-scripts:
-    if : ${{ github.event_name == 'pull_request' }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - uses: ./swift/actions/database-upgrade-scripts

.gitignore

@@ -27,6 +27,8 @@
 # It's useful (though not required) to be able to unpack codeql in the ql checkout itself
 /codeql/
 
+csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
+
 # Avoid committing cached package components
 .codeql
 

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
     rev: v1.6.0
     hooks:
       - id: autopep8
-        files: ^swift/.*\.py
+        files: ^swift/codegen/.*\.py
 
   - repo: local
     hooks:
@@ -31,7 +31,7 @@ repos:
 
       - id: sync-files
         name: Fix files required to be identical
-        files: \.(qll?|qhelp|swift)$|^config/identical-files\.json$
+        files: \.(qll?|qhelp|swift)$
         language: system
         entry: python3 config/sync-files.py --latest
         pass_filenames: false
@@ -44,7 +44,7 @@ repos:
 
       - id: swift-codegen
         name: Run Swift checked in code generation
-        files: ^swift/(schema.py$|codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements)|ql/\.generated.list)
+        files: ^swift/(codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements))
         language: system
         entry: bazel run //swift/codegen -- --quiet
         pass_filenames: false

.vscode/settings.json

@@ -1,5 +1,3 @@
 {
-  "omnisharp.autoStart": false,
-  "cmake.sourceDirectory": "${workspaceFolder}/swift",
-  "cmake.buildDirectory": "${workspaceFolder}/bazel-cmake-build"
+    "omnisharp.autoStart": false
 }

CODEOWNERS

@@ -5,17 +5,24 @@
 /javascript/ @github/codeql-javascript
 /python/ @github/codeql-python
 /ruby/ @github/codeql-ruby
-/swift/ @github/codeql-swift
+/swift/ @github/codeql-c
 /java/kotlin-extractor/ @github/codeql-kotlin
 /java/kotlin-explorer/ @github/codeql-kotlin
 
 # ML-powered queries
 /javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
 
+# Notify members of codeql-go about PRs to the shared data-flow library files
+/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl.qll @github/codeql-java @github/codeql-go
+/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl2.qll @github/codeql-java @github/codeql-go
+/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll @github/codeql-java @github/codeql-go
+/java/ql/src/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll @github/codeql-java @github/codeql-go
+/java/ql/src/semmle/code/java/dataflow/internal/tainttracking2/TaintTrackingImpl.qll @github/codeql-java @github/codeql-go
+
 # CodeQL tools and associated docs
-/docs/codeql/codeql-cli/ @github/codeql-cli-reviewers
-/docs/codeql/codeql-for-visual-studio-code/ @github/codeql-vscode-reviewers
-/docs/codeql/ql-language-reference/ @github/codeql-frontend-reviewers
+/docs/codeql-cli/ @github/codeql-cli-reviewers
+/docs/codeql-for-visual-studio-code/ @github/codeql-vscode-reviewers
+/docs/ql-language-reference/ @github/codeql-frontend-reviewers
 /docs/query-*-style-guide.md @github/codeql-analysis-reviewers
 
 # QL for QL reviewers
@@ -33,9 +40,8 @@ WORKSPACE.bazel @github/codeql-ci-reviewers
 
 # Workflows
 /.github/workflows/ @github/codeql-ci-reviewers
-/.github/workflows/atm-* @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/go-* @github/codeql-go
 /.github/workflows/js-ml-tests.yml @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
 /.github/workflows/ruby-* @github/codeql-ruby
-/.github/workflows/swift.yml @github/codeql-swift
+/.github/workflows/swift-* @github/codeql-c

CONTRIBUTING.md

@@ -25,7 +25,6 @@ If you have an idea for a query that you would like to share with other CodeQL u
 
     Each language-specific directory contains further subdirectories that group queries based on their `@tags` or purpose.
     - Experimental queries and libraries are stored in the `experimental` subdirectory within each language-specific directory in the [CodeQL repository](https://github.com/github/codeql). For example, experimental Java queries and libraries are stored in `java/ql/src/experimental` and any corresponding tests in `java/ql/test/experimental`.
-    - Experimental queries need to include `experimental` in their `@tags`
     - The structure of an `experimental` subdirectory mirrors the structure of its parent directory.
     - Select or create an appropriate directory in `experimental` based on the existing directory structure of `experimental` or its parent directory.
 

README.md

@@ -4,7 +4,8 @@ This open source repository contains the standard CodeQL libraries and queries t
 
 ## How do I learn CodeQL and run queries?
 
-There is [extensive documentation](https://codeql.github.com/docs/) on getting started with writing CodeQL using the [CodeQL extension for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/) and the [CodeQL CLI](https://codeql.github.com/docs/codeql-cli/).
+There is [extensive documentation](https://codeql.github.com/docs/) on getting started with writing CodeQL.
+You can use the [CodeQL for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/) extension or the [interactive query console](https://lgtm.com/help/lgtm/using-query-console) on LGTM.com (Semmle Legacy product) to try out your queries on any open source project that's currently being analyzed.
 
 ## Contributing
 

change-notes/1.20/analysis-javascript.md

@@ -52,7 +52,7 @@
 | Unneeded defensive code | More true positive and fewer false positive results | This query now recognizes additional defensive code patterns. |
 | Unsafe dynamic method access              | Fewer false positive results | This query no longer flags concatenated strings as unsafe method names. |
 | Unused parameter                           | Fewer false positive results | This query no longer flags parameters with leading underscore. |
-| Unused variable, import, function or class | Fewer false positive results | This query now flags fewer variables that are implicitly used by JSX elements. It no longer flags variables with a leading underscore and variables in dead code. |
+| Unused variable, import, function or class | Fewer false positive results | This query now flags fewer variables that are implictly used by JSX elements. It no longer flags variables with a leading underscore and variables in dead code. |
 | Unvalidated dynamic method call           | More true positive results | This query now flags concatenated strings as unvalidated method names in more cases. |
 | Useless assignment to property. | Fewer false positive results | This query now treats assignments with complex right-hand sides correctly. |
 | Useless conditional | Fewer results | Additional defensive coding patterns are now ignored. |

change-notes/1.23/analysis-cpp.md

@@ -19,7 +19,7 @@ The following changes in version 1.23 affect C/C++ analysis in all applications.
 | Hard-coded Japanese era start date in call (`cpp/japanese-era/constructor-or-method-with-exact-era-date`) | Deprecated | This query has been deprecated.  Use the new combined query Hard-coded Japanese era start date (`cpp/japanese-era/exact-era-date`) instead. |
 | Hard-coded Japanese era start date in struct (`cpp/japanese-era/struct-with-exact-era-date`) | Deprecated | This query has been deprecated.  Use the new combined query Hard-coded Japanese era start date (`cpp/japanese-era/exact-era-date`) instead. |
 | Hard-coded Japanese era start date (`cpp/japanese-era/exact-era-date`) | More correct results | This query now checks for the beginning date of the Reiwa era (1st May 2019). |
-| Non-constant format string (`cpp/non-constant-format`) | Fewer false positive results | Fixed false positive results triggered by mismatching declarations of a formatting function. |
+| Non-constant format string (`cpp/non-constant-format`) | Fewer false positive results | Fixed false positive results triggrered by mismatching declarations of a formatting function. |
 | Sign check of bitwise operation (`cpp/bitwise-sign-check`) | Fewer false positive results | Results involving `>=` or `<=` are no longer reported. |
 | Too few arguments to formatting function (`cpp/wrong-number-format-arguments`) | Fewer false positive results | Fixed false positive results triggered by mismatching declarations of a formatting function. |
 | Too many arguments to formatting function (`cpp/too-many-format-arguments`) | Fewer false positive results | Fixed false positive results triggered by mismatching declarations of a formatting function. |

change-notes/1.24/analysis-javascript.md

@@ -91,7 +91,7 @@
 
 ## Changes to libraries
 
-* The predicates `RegExpTerm.getSuccessor` and `RegExpTerm.getPredecessor` have been changed to reflect textual, not operational, matching order. This only makes a difference in lookbehind assertions, which are operationally matched backwards. Previously, `getSuccessor` would mimic this, so in an assertion `(?<=ab)` the term `b` would be considered the predecessor, not the successor, of `a`. Textually, however, `a` is still matched before `b`, and this is the order we now follow.
+* The predicates `RegExpTerm.getSuccessor` and `RegExpTerm.getPredecessor` have been changed to reflect textual, not operational, matching order. This only makes a difference in lookbehind assertions, which are operationally matched backwards. Previously, `getSuccessor` would mimick this, so in an assertion `(?<=ab)` the term `b` would be considered the predecessor, not the successor, of `a`. Textually, however, `a` is still matched before `b`, and this is the order we now follow.
 * An extensible model of the `EventEmitter` pattern has been implemented.
 * Taint-tracking configurations now interact differently with the `data` flow label, which may affect queries
   that combine taint-tracking and flow labels.

codeql-workspace.yml

@@ -17,7 +17,6 @@ provide:
   # - "javascript/ql/experimental/adaptivethreatmodeling/model/qlpack.yml"
   - "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml"
   - "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/test/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/lib/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/src/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/test/qlpack.yml"
@@ -25,8 +24,7 @@ provide:
   - "misc/suite-helpers/qlpack.yml"
   - "ruby/extractor-pack/codeql-extractor.yml"
   - "swift/extractor-pack/codeql-extractor.yml"
-  - "swift/integration-tests/qlpack.yml"
-  - "ql/extractor-pack/codeql-extractor.yml"
+  - "ql/extractor-pack/codeql-extractor.ym"
 
 versionPolicies:
   default:

config/atm/ml-powered-queries-repo/add-note.js

@@ -1,21 +0,0 @@
-const mongoose = require('mongoose');
-
-Logger = require('./logger').Logger;
-Note = require('./models/note').Note;
-
-(async () => {
-  if (process.argv.length != 5) {
-    Logger.log("Creates a private note.  Usage: node add-note.js <token> <title> <body>")
-    return;
-  }
-
-  // Open the default mongoose connection
-  await mongoose.connect('mongodb://localhost:27017/notes', { useFindAndModify: false });
-
-  const [userToken, title, body] = process.argv.slice(2);
-  await Note.create({ title, body, userToken });
-
-  Logger.log(`Created private note with title ${title} and body ${body} belonging to user with token ${userToken}.`);
-
-  await mongoose.connection.close();
-})();

config/atm/ml-powered-queries-repo/app.js

@@ -1,68 +0,0 @@
-const bodyParser = require('body-parser');
-const express = require('express');
-const mongoose = require('mongoose');
-
-const notesApi = require('./notes-api');
-const usersApi = require('./users-api');
-
-const addSampleData = module.exports.addSampleData = async () => {
-  const [userA, userB] = await User.create([
-    {
-      name: "A",
-      token: "tokenA"
-    },
-    {
-      name: "B",
-      token: "tokenB"
-    }
-  ]);
-
-  await Note.create([
-    {
-      title: "Public note belonging to A",
-      body: "This is a public note belonging to A",
-      isPublic: true,
-      ownerToken: userA.token
-    },
-    {
-      title: "Public note belonging to B",
-      body: "This is a public note belonging to B",
-      isPublic: true,
-      ownerToken: userB.token
-    },
-    {
-      title: "Private note belonging to A",
-      body: "This is a private note belonging to A",
-      ownerToken: userA.token
-    },
-    {
-      title: "Private note belonging to B",
-      body: "This is a private note belonging to B",
-      ownerToken: userB.token
-    }
-  ]);
-}
-
-module.exports.startApp = async () => {
-  // Open the default mongoose connection
-  await mongoose.connect('mongodb://mongo:27017/notes', { useFindAndModify: false });
-  // Drop contents of DB
-  mongoose.connection.dropDatabase();
-  // Add some sample data
-  await addSampleData();
-
-  const app = express();
-
-  app.use(bodyParser.json());
-  app.use(bodyParser.urlencoded());
-
-  app.get('/', async (_req, res) => {
-    res.send('Hello World');
-  });
-
-  app.use('/api/notes', notesApi.router);
-  app.use('/api/users', usersApi.router);
-
-  app.listen(3000);
-  Logger.log('Express started on port 3000');
-};

config/atm/ml-powered-queries-repo/index.js

@@ -1,7 +0,0 @@
-const startApp = require('./app').startApp;
-
-Logger = require('./logger').Logger;
-Note = require('./models/note').Note;
-User = require('./models/user').User;
-
-startApp();

config/atm/ml-powered-queries-repo/logger.js

@@ -1,5 +0,0 @@
-module.exports.Logger = class {
-  log(message, ...objs) {
-    console.log(message, objs);
-  }
-};

config/atm/ml-powered-queries-repo/models/note.js

@@ -1,8 +0,0 @@
-const mongoose = require('mongoose');
-
-module.exports.Note = mongoose.model('Note', new mongoose.Schema({
-  title: String,
-  body: String,
-  ownerToken: String,
-  isPublic: Boolean
-}));

config/atm/ml-powered-queries-repo/models/user.js

@@ -1,6 +0,0 @@
-const mongoose = require('mongoose');
-
-module.exports.User = mongoose.model('User', new mongoose.Schema({
-  name: String,
-  token: String
-}));

config/atm/ml-powered-queries-repo/notes-api.js

@@ -1,44 +0,0 @@
-const express = require('express')
-
-const router = module.exports.router = express.Router();
-
-function serializeNote(note) {
-  return {
-    title: note.title,
-    body: note.body
-  };
-}
-
-router.post('/find', async (req, res) => {
-  const notes = await Note.find({
-    ownerToken: req.body.token
-  }).exec();
-  res.json({
-    notes: notes.map(serializeNote)
-  });
-});
-
-router.get('/findPublic', async (_req, res) => {
-  const notes = await Note.find({
-    isPublic: true
-  }).exec();
-  res.json({
-    notes: notes.map(serializeNote)
-  });
-});
-
-router.post('/findVisible', async (req, res) => {
-  const notes = await Note.find({
-    $or: [
-      {
-        isPublic: true
-      },
-      {
-        ownerToken: req.body.token
-      }
-    ]
-  }).exec();
-  res.json({
-    notes: notes.map(serializeNote)
-  });
-});

config/atm/ml-powered-queries-repo/read-notes.js

@@ -1,37 +0,0 @@
-const mongoose = require('mongoose');
-
-Logger = require('./logger').Logger;
-Note = require('./models/note').Note;
-User = require('./models/user').User;
-
-(async () => {
-  if (process.argv.length != 3) {
-    Logger.log("Outputs all notes visible to a user.  Usage: node read-notes.js <token>")
-    return;
-  }
-
-  // Open the default mongoose connection
-  await mongoose.connect('mongodb://localhost:27017/notes', { useFindAndModify: false });
-
-  const ownerToken = process.argv[2];
-
-  const user = await User.findOne({
-    token: ownerToken
-  }).exec();
-
-  const notes = await Note.find({
-    $or: [
-      { isPublic: true },
-      { ownerToken }
-    ]
-  }).exec();
-
-  notes.map(note => {
-    Logger.log("Title:" + note.title);
-    Logger.log("By:" + user.name);
-    Logger.log("Body:" + note.body);
-    Logger.log();
-  });
-
-  await mongoose.connection.close();
-})();

config/atm/ml-powered-queries-repo/users-api.js

@@ -1,25 +0,0 @@
-const express = require('express')
-
-Logger = require('./logger').Logger;
-const router = module.exports.router = express.Router();
-
-router.post('/updateName', async (req, res) => {
-  Logger.log("/updateName called with new name", req.body.name);
-  await User.findOneAndUpdate({
-    token: req.body.token
-  }, {
-    name: req.body.name
-  }).exec();
-  res.json({
-    name: req.body.name
-  });
-});
-
-router.post('/getName', async (req, res) => {
-  const user = await User.findOne({
-    token: req.body.token
-  }).exec();
-  res.json({
-    name: user.name
-  });
-});

config/identical-files.json

@@ -1,5 +1,5 @@
 {
-  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift": [
+  "DataFlow Java/C++/C#/Python": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl2.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl3.qll",
@@ -27,32 +27,27 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplForContentDataFlow.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl2.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImplForStringsNewReplacer.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForRegExp.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForLibraries.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForHttpClientLibraries.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForPathname.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"
   ],
-  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift Common": [
+  "DataFlow Java/C++/C#/Python Common": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
     "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImplCommon.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll"
   ],
-  "TaintTracking::Configuration Java/C++/C#/Go/Python/Ruby/Swift": [
+  "TaintTracking::Configuration Java/C++/C#/Python": [
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
@@ -66,8 +61,6 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking4/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking5/TaintTrackingImpl.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
@@ -76,9 +69,10 @@
     "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking3/TaintTrackingImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking4/TaintTrackingImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttrackingforlibraries/TaintTrackingImpl.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
   ],
-  "DataFlow Java/C++/C#/Python/Ruby/Swift Consistency checks": [
+  "DataFlow Java/C++/C#/Python Consistency checks": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplConsistency.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll",
@@ -88,10 +82,9 @@
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplConsistency.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplConsistency.qll"
   ],
-  "DataFlow Java/C#/Go/Ruby/Python/Swift Flow Summaries": [
+  "DataFlow Java/C#/Ruby/Python/Swift Flow Summaries": [
     "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/FlowSummaryImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/FlowSummaryImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/FlowSummaryImpl.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/FlowSummaryImpl.qll"
@@ -101,8 +94,8 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll"
   ],
   "Model as Data Generation Java/C# - CaptureModels": [
-    "java/ql/src/utils/modelgenerator/internal/CaptureModels.qll",
-    "csharp/ql/src/utils/modelgenerator/internal/CaptureModels.qll"
+    "java/ql/src/utils/model-generator/internal/CaptureModels.qll",
+    "csharp/ql/src/utils/model-generator/internal/CaptureModels.qll"
   ],
   "Sign Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/Sign.qll",
@@ -403,6 +396,16 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/ControlFlowReachability.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/ControlFlowReachability.qll"
   ],
+  "Inline Test Expectations": [
+    "cpp/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "csharp/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "java/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "python/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "ruby/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "ql/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "go/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "swift/ql/test/TestUtilities/InlineExpectationsTest.qll"
+  ],
   "C++ ExternalAPIs": [
     "cpp/ql/src/Security/CWE/CWE-020/ExternalAPIs.qll",
     "cpp/ql/src/Security/CWE/CWE-020/ir/ExternalAPIs.qll"
@@ -461,10 +464,6 @@
     "javascript/ql/src/Comments/CommentedOutCodeReferences.inc.qhelp",
     "python/ql/src/Lexical/CommentedOutCodeReferences.inc.qhelp"
   ],
-  "ThreadResourceAbuse qhelp": [
-    "java/ql/src/experimental/Security/CWE/CWE-400/LocalThreadResourceAbuse.qhelp",
-    "java/ql/src/experimental/Security/CWE/CWE-400/ThreadResourceAbuse.qhelp"
-  ],
   "IDE Contextual Queries": [
     "cpp/ql/lib/IDEContextual.qll",
     "csharp/ql/lib/IDEContextual.qll",
@@ -487,6 +486,40 @@
     "python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll",
     "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll"
   ],
+  "ReDoS Util Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/NfaUtils.qll",
+    "python/ql/lib/semmle/python/security/regexp/NfaUtils.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/NfaUtils.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/NfaUtils.qll"
+  ],
+  "ReDoS Exponential Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/ExponentialBackTracking.qll",
+    "python/ql/lib/semmle/python/security/regexp/ExponentialBackTracking.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/ExponentialBackTracking.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/ExponentialBackTracking.qll"
+  ],
+  "ReDoS Polynomial Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/SuperlinearBackTracking.qll",
+    "python/ql/lib/semmle/python/security/regexp/SuperlinearBackTracking.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/SuperlinearBackTracking.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/SuperlinearBackTracking.qll"
+  ],
+  "RegexpMatching Python/JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/RegexpMatching.qll",
+    "python/ql/lib/semmle/python/security/regexp/RegexpMatching.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/RegexpMatching.qll"
+  ],
+  "BadTagFilterQuery Python/JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll",
+    "python/ql/lib/semmle/python/security/BadTagFilterQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/BadTagFilterQuery.qll"
+  ],
+  "OverlyLargeRange Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/OverlyLargeRangeQuery.qll",
+    "python/ql/lib/semmle/python/security/OverlyLargeRangeQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/OverlyLargeRangeQuery.qll",
+    "java/ql/lib/semmle/code/java/security/OverlyLargeRangeQuery.qll"
+  ],
   "CFG": [
     "csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll",
     "ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll",
@@ -496,9 +529,16 @@
     "python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll",
     "ruby/ql/lib/codeql/ruby/typetracking/TypeTracker.qll"
   ],
+  "CodeQL Tutorial": [
+    "cpp/ql/lib/tutorial.qll",
+    "csharp/ql/lib/tutorial.qll",
+    "java/ql/lib/tutorial.qll",
+    "javascript/ql/lib/tutorial.qll",
+    "python/ql/lib/tutorial.qll",
+    "ruby/ql/lib/tutorial.qll"
+  ],
   "AccessPathSyntax": [
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/AccessPathSyntax.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/AccessPathSyntax.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/AccessPathSyntax.qll",
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/AccessPathSyntax.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll",
@@ -514,16 +554,16 @@
     "ruby/ql/lib/codeql/ruby/internal/ConceptsShared.qll",
     "javascript/ql/lib/semmle/javascript/internal/ConceptsShared.qll"
   ],
+  "Hostname Regexp queries": [
+    "javascript/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
+    "python/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
+    "ruby/ql/src/queries/security/cwe-020/HostnameRegexpShared.qll"
+  ],
   "ApiGraphModels": [
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll",
     "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll",
     "python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll"
   ],
-  "ApiGraphModelsExtensions": [
-    "javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModelsExtensions.qll",
-    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModelsExtensions.qll",
-    "python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsExtensions.qll"
-  ],
   "TaintedFormatStringQuery Ruby/JS": [
     "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringQuery.qll",
     "ruby/ql/lib/codeql/ruby/security/TaintedFormatStringQuery.qll"
@@ -560,16 +600,8 @@
     "swift/ql/test/extractor-tests/patterns/patterns.swift",
     "swift/ql/test/library-tests/ast/patterns.swift"
   ],
-  "Swift control flow test file": [
-    "swift/ql/test/library-tests/controlflow/graph/cfg.swift",
-    "swift/ql/test/library-tests/ast/cfg.swift"
-  ],
   "IncompleteMultiCharacterSanitization JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/IncompleteMultiCharacterSanitizationQuery.qll",
     "ruby/ql/lib/codeql/ruby/security/IncompleteMultiCharacterSanitizationQuery.qll"
-  ],
-  "EncryptionKeySizes Python/Java": [
-    "python/ql/lib/semmle/python/security/internal/EncryptionKeySizes.qll",
-    "java/ql/lib/semmle/code/java/security/internal/EncryptionKeySizes.qll"
   ]
-}
+}

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -257,11 +257,11 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.GetCurrentDirectory = cwd;
             Actions.IsWindows = isWindows;
 
-            var options = new CppAutobuildOptions(Actions);
+            var options = new AutobuildOptions(Actions, Language.Cpp);
             return new CppAutobuilder(Actions, options);
         }
 
-        void TestAutobuilderScript(CppAutobuilder autobuilder, int expectedOutput, int commandsRun)
+        void TestAutobuilderScript(Autobuilder autobuilder, int expectedOutput, int commandsRun)
         {
             Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(Actions, StartCallback, EndCallback));
 
@@ -299,7 +299,7 @@ namespace Semmle.Autobuild.Cpp.Tests
         {
             Actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test.sln -DisableParallelProcessing"] = 1;
             Actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test.sln -DisableParallelProcessing"] = 0;
-            Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && msbuild C:\Project\test.sln /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"""] = 0;
+            Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0;
             Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0;

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj

@@ -11,12 +11,11 @@
   <ItemGroup>
     <PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
     <PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
-    <PackageReference Include="xunit" Version="2.4.2" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
   </ItemGroup>
 
   <ItemGroup>

cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs

@@ -2,26 +2,9 @@
 
 namespace Semmle.Autobuild.Cpp
 {
-    /// <summary>
-    /// Encapsulates C++ build options.
-    /// </summary>
-    public class CppAutobuildOptions : AutobuildOptionsShared
+    public class CppAutobuilder : Autobuilder
     {
-        public override Language Language => Language.Cpp;
-
-
-        /// <summary>
-        /// Reads options from environment variables.
-        /// Throws ArgumentOutOfRangeException for invalid arguments.
-        /// </summary>
-        public CppAutobuildOptions(IBuildActions actions) : base(actions)
-        {
-        }
-    }
-
-    public class CppAutobuilder : Autobuilder<CppAutobuildOptions>
-    {
-        public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options) { }
+        public CppAutobuilder(IBuildActions actions, AutobuildOptions options) : base(actions, options) { }
 
         public override BuildScript GetBuildScript()
         {

cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs

@@ -11,14 +11,14 @@ namespace Semmle.Autobuild.Cpp
             try
             {
                 var actions = SystemBuildActions.Instance;
-                var options = new CppAutobuildOptions(actions);
+                var options = new AutobuildOptions(actions, Language.Cpp);
                 try
                 {
                     Console.WriteLine("CodeQL C++ autobuilder");
                     var builder = new CppAutobuilder(actions, options);
                     return builder.AttemptBuild();
                 }
-                catch (InvalidEnvironmentException ex)
+                catch(InvalidEnvironmentException ex)
                 {
                     Console.WriteLine("The environment is invalid: {0}", ex.Message);
                 }

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -17,7 +17,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Build" Version="17.3.2" />
+    <PackageReference Include="Microsoft.Build" Version="16.11.0" />
   </ItemGroup>
 
   <ItemGroup>

cpp/downgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/exprs.ql

@@ -13,5 +13,5 @@ predicate isExprWithNewBuiltin(Expr expr) {
 from Expr expr, int kind, int kind_new, Location location
 where
   exprs(expr, kind, location) and
-  if isExprWithNewBuiltin(expr) then kind_new = 1 else kind_new = kind
+  if isExprWithNewBuiltin(expr) then kind_new = 0 else kind_new = kind
 select expr, kind_new, location

cpp/downgrades/73af5058c6899dcdb05754c27ca966aeb3a68c94/exprs.ql

@@ -9,5 +9,5 @@ class Location extends @location_expr {
 from Expr expr, int kind, int kind_new, Location location
 where
   exprs(expr, kind, location) and
-  if expr instanceof @blockassignexpr then kind_new = 1 else kind_new = kind
+  if expr instanceof @blockassignexpr then kind_new = 0 else kind_new = kind
 select expr, kind_new, location

cpp/downgrades/a5bb28ed29f73855d64cc5f939cef977fa8fd19a/builtintypes.ql

@@ -1,11 +0,0 @@
-class BuiltinType extends @builtintype {
-  string toString() { none() }
-}
-
-from BuiltinType type, string name, int kind, int kind_new, int size, int sign, int alignment
-where
-  builtintypes(type, name, kind, size, sign, alignment) and
-  if type instanceof @float16 or type instanceof @complex_float16
-  then kind_new = 2
-  else kind_new = kind
-select type, name, kind_new, size, sign, alignment

cpp/downgrades/a5bb28ed29f73855d64cc5f939cef977fa8fd19a/upgrade.properties

@@ -1,3 +0,0 @@
-description: Introduce (_Complex) _Float16 type
-compatibility: backwards
-builtintypes.rel: run builtintypes.qlo

cpp/downgrades/ba86bebea4c7a8235c2fa0e220391fbd4446a087/upgrade.properties

@@ -1,2 +0,0 @@
-description: Uncomment case splits in dbscheme
-compatibility: full

cpp/ql/examples/qlpack.yml

@@ -3,4 +3,4 @@ groups:
   - cpp
   - examples
 dependencies:
-  codeql/cpp-all: ${workspace}
+  codeql/cpp-all: "*"

cpp/ql/lib/CHANGELOG.md

@@ -1,54 +1,3 @@
-## 0.5.1
-
-No user-facing changes.
-
-## 0.5.0
-
-### Breaking Changes
-
-The predicates in the `MustFlow::Configuration` class used by the `MustFlow` library (`semmle.code.cpp.ir.dataflow.MustFlow`) have changed to be defined directly in terms of the C++ IR instead of IR dataflow nodes.
-
-### Deprecated APIs
-
-* Deprecated `semmle.code.cpp.ir.dataflow.DefaultTaintTracking`. Use `semmle.code.cpp.ir.dataflow.TaintTracking`.
-* Deprecated `semmle.code.cpp.security.TaintTrackingImpl`. Use `semmle.code.cpp.ir.dataflow.TaintTracking`.
-* Deprecated `semmle.code.cpp.valuenumbering.GlobalValueNumberingImpl`. Use `semmle.code.cpp.valuenumbering.GlobalValueNumbering`, which exposes the same API.
-
-### Minor Analysis Improvements
-
-* The `ArgvSource` flow source now uses the second parameter of `main` as its source instead of the uses of this parameter.
-* The `ArgvSource` flow source has been generalized to handle cases where the argument vector of `main` is not named `argv`.
-* The `getaddrinfo` function is now recognized as a flow source.
-* The `secure_getenv` and `_wgetenv` functions are now recognized as local flow sources.
-* The `scanf` and `fscanf` functions and their variants are now recognized as flow sources.
-* Deleted the deprecated `getName` and `getShortName` predicates from the `Folder` class.
-
-## 0.4.6
-
-No user-facing changes.
-
-## 0.4.5
-
-No user-facing changes.
-
-## 0.4.4
-
-No user-facing changes.
-
-## 0.4.3
-
-### Minor Analysis Improvements
-
-* Fixed bugs in the `FormatLiteral` class that were causing `getMaxConvertedLength` and related predicates to return no results when the format literal was `%e`, `%f` or `%g` and an explicit precision was specified.
-
-## 0.4.2
-
-No user-facing changes.
-
-## 0.4.1
-
-No user-facing changes.
-
 ## 0.4.0
 
 ### Deprecated APIs

cpp/ql/lib/change-notes/released/0.4.1.md

@@ -1,3 +0,0 @@
-## 0.4.1
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.4.2.md

@@ -1,3 +0,0 @@
-## 0.4.2
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.4.3.md

@@ -1,5 +0,0 @@
-## 0.4.3
-
-### Minor Analysis Improvements
-
-* Fixed bugs in the `FormatLiteral` class that were causing `getMaxConvertedLength` and related predicates to return no results when the format literal was `%e`, `%f` or `%g` and an explicit precision was specified.

cpp/ql/lib/change-notes/released/0.4.4.md

@@ -1,3 +0,0 @@
-## 0.4.4
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.4.5.md

@@ -1,3 +0,0 @@
-## 0.4.5
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.4.6.md

@@ -1,3 +0,0 @@
-## 0.4.6
-
-No user-facing changes.

cpp/ql/lib/change-notes/released/0.5.0.md

@@ -1,20 +0,0 @@
-## 0.5.0
-
-### Breaking Changes
-
-The predicates in the `MustFlow::Configuration` class used by the `MustFlow` library (`semmle.code.cpp.ir.dataflow.MustFlow`) have changed to be defined directly in terms of the C++ IR instead of IR dataflow nodes.
-
-### Deprecated APIs
-
-* Deprecated `semmle.code.cpp.ir.dataflow.DefaultTaintTracking`. Use `semmle.code.cpp.ir.dataflow.TaintTracking`.
-* Deprecated `semmle.code.cpp.security.TaintTrackingImpl`. Use `semmle.code.cpp.ir.dataflow.TaintTracking`.
-* Deprecated `semmle.code.cpp.valuenumbering.GlobalValueNumberingImpl`. Use `semmle.code.cpp.valuenumbering.GlobalValueNumbering`, which exposes the same API.
-
-### Minor Analysis Improvements
-
-* The `ArgvSource` flow source now uses the second parameter of `main` as its source instead of the uses of this parameter.
-* The `ArgvSource` flow source has been generalized to handle cases where the argument vector of `main` is not named `argv`.
-* The `getaddrinfo` function is now recognized as a flow source.
-* The `secure_getenv` and `_wgetenv` functions are now recognized as local flow sources.
-* The `scanf` and `fscanf` functions and their variants are now recognized as flow sources.
-* Deleted the deprecated `getName` and `getShortName` predicates from the `Folder` class.

cpp/ql/lib/change-notes/released/0.5.1.md

@@ -1,3 +0,0 @@
-## 0.5.1
-
-No user-facing changes.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.5.1
+lastReleaseVersion: 0.4.0

cpp/ql/lib/definitions.qll

@@ -12,8 +12,8 @@ import IDEContextual
  *
  * In some cases it is preferable to modify locations (the
  * `hasLocationInfo()` predicate) so that they are short, and
- * non-overlapping with other locations that might be reported as
- * code scanning alerts on GitHub.
+ * non-overlapping with other locations that might be highlighted in
+ * the LGTM interface.
  *
  * We need to give locations that may not be in the database, so
  * we use `hasLocationInfo()` rather than `getLocation()`.
@@ -123,13 +123,6 @@ private predicate constructorCallTypeMention(ConstructorCall cc, TypeMention tm)
   )
 }
 
-/** Holds if `loc` has the container `container` and is on the line starting at `startLine`. */
-pragma[nomagic]
-private predicate hasContainerAndStartLine(Location loc, Container container, int startLine) {
-  loc.getStartLine() = startLine and
-  loc.getContainer() = container
-}
-
 /**
  * Gets an element, of kind `kind`, that element `e` uses, if any.
  * Attention: This predicate yields multiple definitions for a single location.
@@ -166,9 +159,9 @@ Top definitionOf(Top e, string kind) {
     // Multiple type mentions can be generated when a typedef is used, and
     // in such cases we want to exclude all but the originating typedef.
     not exists(Type secondary |
-      exists(File f, int startline, int startcol |
+      exists(TypeMention tm, File f, int startline, int startcol |
         typeMentionStartLoc(e, result, f, startline, startcol) and
-        typeMentionStartLoc(_, secondary, f, startline, startcol) and
+        typeMentionStartLoc(tm, secondary, f, startline, startcol) and
         (
           result = secondary.(TypedefType).getBaseType() or
           result = secondary.(TypedefType).getBaseType().(SpecifiedType).getBaseType()
@@ -191,9 +184,11 @@ Top definitionOf(Top e, string kind) {
     kind = "I" and
     result = e.(Include).getIncludedFile() and
     // exclude `#include` directives containing macros
-    not exists(MacroInvocation mi, Container container, int startLine |
-      hasContainerAndStartLine(e.(Include).getLocation(), container, startLine) and
-      hasContainerAndStartLine(mi.getLocation(), container, startLine)
+    not exists(MacroInvocation mi, Location l1, Location l2 |
+      l1 = e.(Include).getLocation() and
+      l2 = mi.getLocation() and
+      l1.getContainer() = l2.getContainer() and
+      l1.getStartLine() = l2.getStartLine()
       // (an #include directive must be always on it's own line)
     )
   ) and

cpp/ql/lib/experimental/semmle/code/cpp/dataflow/ProductFlow.qll

@@ -20,8 +20,7 @@ module ProductFlow {
      * `source1` and `source2` must belong to the same callable.
      */
     predicate isSourcePair(
-      DataFlow::Node source1, DataFlow::FlowState state1, DataFlow::Node source2,
-      DataFlow::FlowState state2
+      DataFlow::Node source1, string state1, DataFlow::Node source2, string state2
     ) {
       state1 = "" and
       state2 = "" and
@@ -90,61 +89,6 @@ module ProductFlow {
      */
     predicate isBarrierOut2(DataFlow::Node node) { none() }
 
-    /*
-     * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps in
-     * the first projection of the product dataflow graph.
-     */
-
-    predicate isAdditionalFlowStep1(DataFlow::Node node1, DataFlow::Node node2) { none() }
-
-    /**
-     * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps in
-     * the first projection of the product dataflow graph.
-     *
-     * This step is only applicable in `state1` and updates the flow state to `state2`.
-     */
-    predicate isAdditionalFlowStep1(
-      DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,
-      DataFlow::FlowState state2
-    ) {
-      state1 instanceof DataFlow::FlowStateEmpty and
-      state2 instanceof DataFlow::FlowStateEmpty and
-      this.isAdditionalFlowStep1(node1, node2)
-    }
-
-    /**
-     * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps in
-     * the second projection of the product dataflow graph.
-     */
-    predicate isAdditionalFlowStep2(DataFlow::Node node1, DataFlow::Node node2) { none() }
-
-    /**
-     * Holds if data may flow from `node1` to `node2` in addition to the normal data-flow steps in
-     * the second projection of the product dataflow graph.
-     *
-     * This step is only applicable in `state1` and updates the flow state to `state2`.
-     */
-    predicate isAdditionalFlowStep2(
-      DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,
-      DataFlow::FlowState state2
-    ) {
-      state1 instanceof DataFlow::FlowStateEmpty and
-      state2 instanceof DataFlow::FlowStateEmpty and
-      this.isAdditionalFlowStep2(node1, node2)
-    }
-
-    /**
-     * Holds if data flow into `node` is prohibited in the first projection of the product
-     * dataflow graph.
-     */
-    predicate isBarrierIn1(DataFlow::Node node) { none() }
-
-    /**
-     * Holds if data flow into `node` is prohibited in the second projection of the product
-     * dataflow graph.
-     */
-    predicate isBarrierIn2(DataFlow::Node node) { none() }
-
     predicate hasFlowPath(
       DataFlow::PathNode source1, DataFlow2::PathNode source2, DataFlow::PathNode sink1,
       DataFlow2::PathNode sink2
@@ -159,78 +103,54 @@ module ProductFlow {
     class Conf1 extends DataFlow::Configuration {
       Conf1() { this = "Conf1" }
 
-      override predicate isSource(DataFlow::Node source, DataFlow::FlowState state) {
+      override predicate isSource(DataFlow::Node source, string state) {
         exists(Configuration conf | conf.isSourcePair(source, state, _, _))
       }
 
-      override predicate isSink(DataFlow::Node sink, DataFlow::FlowState state) {
+      override predicate isSink(DataFlow::Node sink, string state) {
         exists(Configuration conf | conf.isSinkPair(sink, state, _, _))
       }
 
-      override predicate isBarrier(DataFlow::Node node, DataFlow::FlowState state) {
+      override predicate isBarrier(DataFlow::Node node, string state) {
         exists(Configuration conf | conf.isBarrier1(node, state))
       }
 
       override predicate isBarrierOut(DataFlow::Node node) {
         exists(Configuration conf | conf.isBarrierOut1(node))
       }
-
-      override predicate isAdditionalFlowStep(
-        DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,
-        DataFlow::FlowState state2
-      ) {
-        exists(Configuration conf | conf.isAdditionalFlowStep1(node1, state1, node2, state2))
-      }
-
-      override predicate isBarrierIn(DataFlow::Node node) {
-        exists(Configuration conf | conf.isBarrierIn1(node))
-      }
     }
 
     class Conf2 extends DataFlow2::Configuration {
       Conf2() { this = "Conf2" }
 
-      override predicate isSource(DataFlow::Node source, DataFlow::FlowState state) {
-        exists(Configuration conf, DataFlow::PathNode source1 |
-          conf.isSourcePair(source1.getNode(), source1.getState(), source, state) and
-          any(Conf1 c).hasFlowPath(source1, _)
+      override predicate isSource(DataFlow::Node source, string state) {
+        exists(Configuration conf, DataFlow::Node source1 |
+          conf.isSourcePair(source1, _, source, state) and
+          any(Conf1 c).hasFlow(source1, _)
         )
       }
 
-      override predicate isSink(DataFlow::Node sink, DataFlow::FlowState state) {
-        exists(Configuration conf, DataFlow::PathNode sink1 |
-          conf.isSinkPair(sink1.getNode(), sink1.getState(), sink, state) and
-          any(Conf1 c).hasFlowPath(_, sink1)
+      override predicate isSink(DataFlow::Node sink, string state) {
+        exists(Configuration conf, DataFlow::Node sink1 |
+          conf.isSinkPair(sink1, _, sink, state) and any(Conf1 c).hasFlow(_, sink1)
         )
       }
 
-      override predicate isBarrier(DataFlow::Node node, DataFlow::FlowState state) {
+      override predicate isBarrier(DataFlow::Node node, string state) {
         exists(Configuration conf | conf.isBarrier2(node, state))
       }
 
       override predicate isBarrierOut(DataFlow::Node node) {
         exists(Configuration conf | conf.isBarrierOut2(node))
       }
-
-      override predicate isAdditionalFlowStep(
-        DataFlow::Node node1, DataFlow::FlowState state1, DataFlow::Node node2,
-        DataFlow::FlowState state2
-      ) {
-        exists(Configuration conf | conf.isAdditionalFlowStep2(node1, state1, node2, state2))
-      }
-
-      override predicate isBarrierIn(DataFlow::Node node) {
-        exists(Configuration conf | conf.isBarrierIn2(node))
-      }
     }
   }
 
-  pragma[nomagic]
   private predicate reachableInterprocEntry(
     Configuration conf, DataFlow::PathNode source1, DataFlow2::PathNode source2,
     DataFlow::PathNode node1, DataFlow2::PathNode node2
   ) {
-    conf.isSourcePair(node1.getNode(), node1.getState(), node2.getNode(), node2.getState()) and
+    conf.isSourcePair(node1.getNode(), _, node2.getNode(), _) and
     node1 = source1 and
     node2 = source2
     or
@@ -293,7 +213,7 @@ module ProductFlow {
   ) {
     exists(DataFlow::PathNode mid1, DataFlow2::PathNode mid2 |
       reachableInterprocEntry(conf, source1, source2, mid1, mid2) and
-      conf.isSinkPair(sink1.getNode(), sink1.getState(), sink2.getNode(), sink2.getState()) and
+      conf.isSinkPair(sink1.getNode(), _, sink2.getNode(), _) and
       localPathStep1*(mid1, sink1) and
       localPathStep2*(mid2, sink2)
     )

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll

@@ -915,57 +915,18 @@ private module Cached {
     TDataFlowCallNone() or
     TDataFlowCallSome(DataFlowCall call)
 
-  cached
-  newtype TParamNodeOption =
-    TParamNodeNone() or
-    TParamNodeSome(ParamNode p)
-
-  cached
-  newtype TReturnCtx =
-    TReturnCtxNone() or
-    TReturnCtxNoFlowThrough() or
-    TReturnCtxMaybeFlowThrough(ReturnPosition pos)
-
-  cached
-  newtype TTypedContentApprox =
-    MkTypedContentApprox(ContentApprox c, DataFlowType t) {
-      exists(Content cont |
-        c = getContentApprox(cont) and
-        store(_, cont, _, _, t)
-      )
-    }
-
   cached
   newtype TTypedContent = MkTypedContent(Content c, DataFlowType t) { store(_, c, _, _, t) }
 
-  cached
-  TypedContent getATypedContent(TypedContentApprox c) {
-    exists(ContentApprox cls, DataFlowType t, Content cont |
-      c = MkTypedContentApprox(cls, pragma[only_bind_into](t)) and
-      result = MkTypedContent(cont, pragma[only_bind_into](t)) and
-      cls = getContentApprox(cont)
-    )
-  }
-
   cached
   newtype TAccessPathFront =
     TFrontNil(DataFlowType t) or
     TFrontHead(TypedContent tc)
 
-  cached
-  newtype TApproxAccessPathFront =
-    TApproxFrontNil(DataFlowType t) or
-    TApproxFrontHead(TypedContentApprox tc)
-
   cached
   newtype TAccessPathFrontOption =
     TAccessPathFrontNone() or
     TAccessPathFrontSome(AccessPathFront apf)
-
-  cached
-  newtype TApproxAccessPathFrontOption =
-    TApproxAccessPathFrontNone() or
-    TApproxAccessPathFrontSome(ApproxAccessPathFront apf)
 }
 
 /**
@@ -1343,113 +1304,6 @@ class DataFlowCallOption extends TDataFlowCallOption {
   }
 }
 
-/** An optional `ParamNode`. */
-class ParamNodeOption extends TParamNodeOption {
-  string toString() {
-    this = TParamNodeNone() and
-    result = "(none)"
-    or
-    exists(ParamNode p |
-      this = TParamNodeSome(p) and
-      result = p.toString()
-    )
-  }
-}
-
-/**
- * A return context used to calculate flow summaries in reverse flow.
- *
- * The possible values are:
- *
- * - `TReturnCtxNone()`: no return flow.
- * - `TReturnCtxNoFlowThrough()`: return flow, but flow through is not possible.
- * - `TReturnCtxMaybeFlowThrough(ReturnPosition pos)`: return flow, of kind `pos`, and
- *    flow through may be possible.
- */
-class ReturnCtx extends TReturnCtx {
-  string toString() {
-    this = TReturnCtxNone() and
-    result = "(none)"
-    or
-    this = TReturnCtxNoFlowThrough() and
-    result = "(no flow through)"
-    or
-    exists(ReturnPosition pos |
-      this = TReturnCtxMaybeFlowThrough(pos) and
-      result = pos.toString()
-    )
-  }
-}
-
-/** An approximated `Content` tagged with the type of a containing object. */
-class TypedContentApprox extends MkTypedContentApprox {
-  private ContentApprox c;
-  private DataFlowType t;
-
-  TypedContentApprox() { this = MkTypedContentApprox(c, t) }
-
-  /** Gets a typed content approximated by this value. */
-  TypedContent getATypedContent() { result = getATypedContent(this) }
-
-  /** Gets the container type. */
-  DataFlowType getContainerType() { result = t }
-
-  /** Gets a textual representation of this approximated content. */
-  string toString() { result = c.toString() }
-}
-
-/**
- * The front of an approximated access path. This is either a head or a nil.
- */
-abstract class ApproxAccessPathFront extends TApproxAccessPathFront {
-  abstract string toString();
-
-  abstract DataFlowType getType();
-
-  abstract boolean toBoolNonEmpty();
-
-  pragma[nomagic]
-  TypedContent getAHead() {
-    exists(TypedContentApprox cont |
-      this = TApproxFrontHead(cont) and
-      result = cont.getATypedContent()
-    )
-  }
-}
-
-class ApproxAccessPathFrontNil extends ApproxAccessPathFront, TApproxFrontNil {
-  private DataFlowType t;
-
-  ApproxAccessPathFrontNil() { this = TApproxFrontNil(t) }
-
-  override string toString() { result = ppReprType(t) }
-
-  override DataFlowType getType() { result = t }
-
-  override boolean toBoolNonEmpty() { result = false }
-}
-
-class ApproxAccessPathFrontHead extends ApproxAccessPathFront, TApproxFrontHead {
-  private TypedContentApprox tc;
-
-  ApproxAccessPathFrontHead() { this = TApproxFrontHead(tc) }
-
-  override string toString() { result = tc.toString() }
-
-  override DataFlowType getType() { result = tc.getContainerType() }
-
-  override boolean toBoolNonEmpty() { result = true }
-}
-
-/** An optional approximated access path front. */
-class ApproxAccessPathFrontOption extends TApproxAccessPathFrontOption {
-  string toString() {
-    this = TApproxAccessPathFrontNone() and result = "<none>"
-    or
-    this = TApproxAccessPathFrontSome(any(ApproxAccessPathFront apf | result = apf.toString()))
-  }
-}
-
 /** A `Content` tagged with the type of a containing object. */
 class TypedContent extends MkTypedContent {
   private Content c;
@@ -1482,7 +1336,7 @@ abstract class AccessPathFront extends TAccessPathFront {
 
   abstract DataFlowType getType();
 
-  abstract ApproxAccessPathFront toApprox();
+  abstract boolean toBoolNonEmpty();
 
   TypedContent getHead() { this = TFrontHead(result) }
 }
@@ -1496,7 +1350,7 @@ class AccessPathFrontNil extends AccessPathFront, TFrontNil {
 
   override DataFlowType getType() { result = t }
 
-  override ApproxAccessPathFront toApprox() { result = TApproxFrontNil(t) }
+  override boolean toBoolNonEmpty() { result = false }
 }
 
 class AccessPathFrontHead extends AccessPathFront, TFrontHead {
@@ -1508,7 +1362,7 @@ class AccessPathFrontHead extends AccessPathFront, TFrontHead {
 
   override DataFlowType getType() { result = tc.getContainerType() }
 
-  override ApproxAccessPathFront toApprox() { result.getAHead() = tc }
+  override boolean toBoolNonEmpty() { result = true }
 }
 
 /** An optional access path front. */

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll

@@ -45,16 +45,6 @@ module Consistency {
     ) {
       none()
     }
-
-    /** Holds if `(c, pos, p)` should be excluded from the consistency test `uniqueParameterNodeAtPosition`. */
-    predicate uniqueParameterNodeAtPositionExclude(DataFlowCallable c, ParameterPosition pos, Node p) {
-      none()
-    }
-
-    /** Holds if `(c, pos, p)` should be excluded from the consistency test `uniqueParameterNodePosition`. */
-    predicate uniqueParameterNodePositionExclude(DataFlowCallable c, ParameterPosition pos, Node p) {
-      none()
-    }
   }
 
   private class RelevantNode extends Node {
@@ -111,7 +101,9 @@ module Consistency {
     exists(int c |
       c =
         strictcount(Node n |
-          not n.hasLocationInfo(_, _, _, _, _) and
+          not exists(string filepath, int startline, int startcolumn, int endline, int endcolumn |
+            n.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+          ) and
           not any(ConsistencyConfiguration conf).missingLocationExclude(n)
         ) and
       msg = "Nodes without location: " + c
@@ -144,18 +136,6 @@ module Consistency {
     msg = "Local flow step does not preserve enclosing callable."
   }
 
-  query predicate readStepIsLocal(Node n1, Node n2, string msg) {
-    readStep(n1, _, n2) and
-    nodeGetEnclosingCallable(n1) != nodeGetEnclosingCallable(n2) and
-    msg = "Read step does not preserve enclosing callable."
-  }
-
-  query predicate storeStepIsLocal(Node n1, Node n2, string msg) {
-    storeStep(n1, _, n2) and
-    nodeGetEnclosingCallable(n1) != nodeGetEnclosingCallable(n2) and
-    msg = "Store step does not preserve enclosing callable."
-  }
-
   private DataFlowType typeRepr() { result = getNodeType(_) }
 
   query predicate compatibleTypesReflexive(DataFlowType t, string msg) {
@@ -252,27 +232,4 @@ module Consistency {
     not callable = viableCallable(call) and
     not any(ConsistencyConfiguration c).viableImplInCallContextTooLargeExclude(call, ctx, callable)
   }
-
-  query predicate uniqueParameterNodeAtPosition(
-    DataFlowCallable c, ParameterPosition pos, Node p, string msg
-  ) {
-    not any(ConsistencyConfiguration conf).uniqueParameterNodeAtPositionExclude(c, pos, p) and
-    isParameterNode(p, c, pos) and
-    not exists(unique(Node p0 | isParameterNode(p0, c, pos))) and
-    msg = "Parameters with overlapping positions."
-  }
-
-  query predicate uniqueParameterNodePosition(
-    DataFlowCallable c, ParameterPosition pos, Node p, string msg
-  ) {
-    not any(ConsistencyConfiguration conf).uniqueParameterNodePositionExclude(c, pos, p) and
-    isParameterNode(p, c, pos) and
-    not exists(unique(ParameterPosition pos0 | isParameterNode(p, c, pos0))) and
-    msg = "Parameter node with multiple positions."
-  }
-
-  query predicate uniqueContentApprox(Content c, string msg) {
-    not exists(unique(ContentApprox approx | approx = getContentApprox(c))) and
-    msg = "Non-unique content approximation."
-  }
 }

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll

@@ -137,7 +137,7 @@ private newtype TReturnKind =
     exists(IndirectReturnNode return, ReturnIndirectionInstruction returnInd |
       returnInd.hasIndex(argumentIndex) and
       return.getAddressOperand() = returnInd.getSourceAddressOperand() and
-      indirectionIndex = return.getIndirectionIndex()
+      indirectionIndex = return.getIndirectionIndex() - 1 // We subtract one because the return loads the value.
     )
   }
 
@@ -197,7 +197,7 @@ class ReturnIndirectionNode extends IndirectReturnNode, ReturnNode {
     exists(int argumentIndex, ReturnIndirectionInstruction returnInd |
       returnInd.hasIndex(argumentIndex) and
       this.getAddressOperand() = returnInd.getSourceAddressOperand() and
-      result = TIndirectReturnKind(argumentIndex, this.getIndirectionIndex()) and
+      result = TIndirectReturnKind(argumentIndex, this.getIndirectionIndex() - 1) and
       hasNonInitializeParameterDef(returnInd.getIRVariable())
     )
     or
@@ -241,7 +241,7 @@ private Instruction getANonConversionUse(Operand operand) {
 
 /**
  * Gets the operand that represents the first use of the value of `call` following
- * a sequence of conversion-like instructions.
+ * a sequnce of conversion-like instructions.
  */
 predicate operandForfullyConvertedCall(Operand operand, CallInstruction call) {
   exists(getANonConversionUse(operand)) and
@@ -254,7 +254,7 @@ predicate operandForfullyConvertedCall(Operand operand, CallInstruction call) {
 
 /**
  * Gets the instruction that represents the first use of the value of `call` following
- * a sequence of conversion-like instructions.
+ * a sequnce of conversion-like instructions.
  *
  * This predicate only holds if there is no suitable operand (i.e., no operand of a non-
  * conversion instruction) to use to represent the value of `call` after conversions.
@@ -365,7 +365,7 @@ predicate jumpStep(Node n1, Node n2) {
 predicate storeStep(Node node1, Content c, PostFieldUpdateNode node2) {
   exists(int indirectionIndex1, int numberOfLoads, StoreInstruction store |
     nodeHasInstruction(node1, store, pragma[only_bind_into](indirectionIndex1)) and
-    node2.getIndirectionIndex() = 1 and
+    node2.getIndirectionIndex() = 0 and
     numberOfLoadsFromOperand(node2.getFieldAddress(), store.getDestinationAddressOperand(),
       numberOfLoads)
   |
@@ -465,20 +465,20 @@ predicate clearsContent(Node n, Content c) {
 predicate expectsContent(Node n, ContentSet c) { none() }
 
 /** Gets the type of `n` used for type pruning. */
-DataFlowType getNodeType(Node n) {
+IRType getNodeType(Node n) {
   suppressUnusedNode(n) and
-  result instanceof VoidType // stub implementation
+  result instanceof IRVoidType // stub implementation
 }
 
 /** Gets a string representation of a type returned by `getNodeType`. */
-string ppReprType(DataFlowType t) { none() } // stub implementation
+string ppReprType(IRType t) { none() } // stub implementation
 
 /**
  * Holds if `t1` and `t2` are compatible, that is, whether data can flow from
  * a node of type `t1` to a node of type `t2`.
  */
 pragma[inline]
-predicate compatibleTypes(DataFlowType t1, DataFlowType t2) {
+predicate compatibleTypes(IRType t1, IRType t2) {
   any() // stub implementation
 }
 
@@ -502,7 +502,7 @@ class DataFlowCallable = Cpp::Declaration;
 
 class DataFlowExpr = Expr;
 
-class DataFlowType = Type;
+class DataFlowType = IRType;
 
 /** A function call relevant for data flow. */
 class DataFlowCall extends CallInstruction {
@@ -551,13 +551,6 @@ predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preserves
  */
 predicate allowParameterReturnInSelf(ParameterNode p) { none() }
 
-/** An approximated `Content`. */
-class ContentApprox = Unit;
-
-/** Gets an approximated value for content `c`. */
-pragma[inline]
-ContentApprox getContentApprox(Content c) { any() }
-
 private class MyConsistencyConfiguration extends Consistency::ConsistencyConfiguration {
   override predicate argHasPostUpdateExclude(ArgumentNode n) {
     // The rules for whether an IR argument gets a post-update node are too

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll

@@ -38,12 +38,13 @@ private module Cached {
     TVariableNode(Variable var) or
     TPostFieldUpdateNode(FieldAddress operand, int indirectionIndex) {
       indirectionIndex =
-        [1 .. Ssa::countIndirectionsForCppType(operand.getObjectAddress().getResultLanguageType())]
+        [0 .. Ssa::countIndirectionsForCppType(operand.getObjectAddress().getResultLanguageType()) -
+            1]
     } or
     TSsaPhiNode(Ssa::PhiNode phi) or
     TIndirectArgumentOutNode(ArgumentOperand operand, int indirectionIndex) {
       Ssa::isModifiableByCall(operand) and
-      indirectionIndex = [1 .. Ssa::countIndirectionsForCppType(operand.getLanguageType())]
+      indirectionIndex = [0 .. Ssa::countIndirectionsForCppType(operand.getLanguageType()) - 1]
     } or
     TIndirectOperand(Operand op, int indirectionIndex) {
       Ssa::hasIndirectOperand(op, indirectionIndex)
@@ -112,7 +113,7 @@ class Node extends TIRDataFlowNode {
   Declaration getFunction() { none() } // overridden in subclasses
 
   /** Gets the type of this node. */
-  DataFlowType getType() { none() } // overridden in subclasses
+  IRType getType() { none() } // overridden in subclasses
 
   /** Gets the instruction corresponding to this node, if any. */
   Instruction asInstruction() { result = this.(InstructionNode).getInstruction() }
@@ -229,13 +230,7 @@ class Node extends TIRDataFlowNode {
   Expr asIndirectArgument() { result = this.asIndirectArgument(_) }
 
   /** Gets the positional parameter corresponding to this node, if any. */
-  Parameter asParameter() { result = this.asParameter(0) }
-
-  /**
-   * Gets the uninitialized local variable corresponding to this node, if
-   * any.
-   */
-  LocalVariable asUninitialized() { result = this.(UninitializedNode).getLocalVariable() }
+  Parameter asParameter() { result = asParameter(0) }
 
   /**
    * Gets the positional parameter corresponding to the node that represents
@@ -278,7 +273,7 @@ class Node extends TIRDataFlowNode {
   /**
    * Gets an upper bound on the type of this node.
    */
-  DataFlowType getTypeBound() { result = this.getType() }
+  IRType getTypeBound() { result = this.getType() }
 
   /** Gets the location of this element. */
   cached
@@ -327,7 +322,7 @@ class InstructionNode extends Node, TInstructionNode {
 
   override Declaration getFunction() { result = instr.getEnclosingFunction() }
 
-  override DataFlowType getType() { result = instr.getResultType() }
+  override IRType getType() { result = instr.getResultIRType() }
 
   final override Location getLocationImpl() { result = instr.getLocation() }
 
@@ -353,32 +348,13 @@ class OperandNode extends Node, TOperandNode {
 
   override Declaration getFunction() { result = op.getUse().getEnclosingFunction() }
 
-  override DataFlowType getType() { result = op.getType() }
+  override IRType getType() { result = op.getIRType() }
 
   final override Location getLocationImpl() { result = op.getLocation() }
 
   override string toStringImpl() { result = this.getOperand().toString() }
 }
 
-/**
- * Returns `t`, but stripped of the `n` outermost pointers, references, etc.
- *
- * For example, `stripPointers(int*&, 2)` is `int` and `stripPointers(int*, 0)` is `int*`.
- */
-private Type stripPointers(Type t, int n) {
-  result = t and n = 0
-  or
-  result = stripPointers(t.(PointerType).getBaseType(), n - 1)
-  or
-  result = stripPointers(t.(ArrayType).getBaseType(), n - 1)
-  or
-  result = stripPointers(t.(ReferenceType).getBaseType(), n - 1)
-  or
-  result = stripPointers(t.(PointerToMemberType).getBaseType(), n - 1)
-  or
-  result = stripPointers(t.(FunctionPointerIshType).getBaseType(), n - 1)
-}
-
 /**
  * INTERNAL: do not use.
  *
@@ -394,6 +370,8 @@ class PostFieldUpdateNode extends TPostFieldUpdateNode, PartialDefinitionNode {
 
   override Declaration getEnclosingCallable() { result = this.getFunction() }
 
+  override IRType getType() { result = fieldAddress.getIRType() }
+
   FieldAddress getFieldAddress() { result = fieldAddress }
 
   Field getUpdatedField() { result = fieldAddress.getField() }
@@ -401,8 +379,10 @@ class PostFieldUpdateNode extends TPostFieldUpdateNode, PartialDefinitionNode {
   int getIndirectionIndex() { result = indirectionIndex }
 
   override Node getPreUpdateNode() {
+    // + 1 because we're storing into an lvalue, and the original node should be the rvalue of
+    // the same address.
     hasOperandAndIndex(result, pragma[only_bind_into](fieldAddress).getObjectAddressOperand(),
-      indirectionIndex)
+      indirectionIndex + 1)
   }
 
   override Expr getDefinedExpr() {
@@ -431,7 +411,7 @@ class SsaPhiNode extends Node, TSsaPhiNode {
 
   override Declaration getFunction() { result = phi.getBasicBlock().getEnclosingFunction() }
 
-  override DataFlowType getType() { result = this.getAnInput().getType() }
+  override IRType getType() { result instanceof IRVoidType }
 
   final override Location getLocationImpl() { result = phi.getBasicBlock().getLocation() }
 
@@ -474,6 +454,8 @@ class SideEffectOperandNode extends Node, IndirectOperand {
 
   override Function getFunction() { result = call.getEnclosingFunction() }
 
+  override IRType getType() { result instanceof IRVoidType }
+
   Expr getArgument() { result = call.getArgument(argumentIndex).getUnconvertedResultExpression() }
 }
 
@@ -496,6 +478,8 @@ class IndirectParameterNode extends Node, IndirectInstruction {
 
   override Function getFunction() { result = this.getInstruction().getEnclosingFunction() }
 
+  override IRType getType() { result instanceof IRVoidType }
+
   override string toStringImpl() {
     result = this.getParameter().toString() + " indirection"
     or
@@ -520,6 +504,8 @@ class IndirectReturnNode extends IndirectOperand {
   Operand getAddressOperand() { result = operand }
 
   override Declaration getEnclosingCallable() { result = this.getFunction() }
+
+  override IRType getType() { result instanceof IRVoidType }
 }
 
 /**
@@ -550,7 +536,9 @@ class IndirectArgumentOutNode extends Node, TIndirectArgumentOutNode, PostUpdate
 
   override Function getFunction() { result = this.getCallInstruction().getEnclosingFunction() }
 
-  override Node getPreUpdateNode() { hasOperandAndIndex(result, operand, indirectionIndex) }
+  override IRType getType() { result instanceof IRVoidType }
+
+  override Node getPreUpdateNode() { hasOperandAndIndex(result, operand, indirectionIndex + 1) }
 
   override string toStringImpl() {
     // This string should be unique enough to be helpful but common enough to
@@ -606,38 +594,6 @@ class IndirectReturnOutNode extends Node {
   int getIndirectionIndex() { result = indirectionIndex }
 }
 
-private PointerType getGLValueType(Type t, int indirectionIndex) {
-  result.getBaseType() = stripPointers(t, indirectionIndex - 1)
-}
-
-bindingset[isGLValue]
-private DataFlowType getTypeImpl(Type t, int indirectionIndex, boolean isGLValue) {
-  if isGLValue = true
-  then
-    result = getGLValueType(t, indirectionIndex)
-    or
-    // Ideally, the above case would cover all glvalue cases. However, consider the case where
-    // the database consists only of:
-    // ```
-    // void test() {
-    //   int* x;
-    //   x = nullptr;
-    // }
-    // ```
-    // and we want to compute the type of `*x` in the assignment `x = nullptr`. Here, `x` is an lvalue
-    // of type int* (which morally is an int**). So when we call `getTypeImpl` it will be with the
-    // parameters:
-    // - t = int*
-    // - indirectionIndex = 1 (when we want to model the dataflow node corresponding to *x)
-    // - isGLValue = true
-    // In this case, `getTypeImpl(t, indirectionIndex, isGLValue)` should give back `int**`. In this
-    // case, however, `int**` does not exist in the database. So instead we return int* (which is
-    // wrong, but at least we have a type).
-    not exists(getGLValueType(t, indirectionIndex)) and
-    result = stripPointers(t, indirectionIndex - 1)
-  else result = stripPointers(t, indirectionIndex)
-}
-
 /**
  * INTERNAL: Do not use.
  *
@@ -659,11 +615,7 @@ class IndirectOperand extends Node, TIndirectOperand {
 
   override Declaration getEnclosingCallable() { result = this.getFunction() }
 
-  override DataFlowType getType() {
-    exists(boolean isGLValue | if operand.isGLValue() then isGLValue = true else isGLValue = false |
-      result = getTypeImpl(operand.getType().getUnspecifiedType(), indirectionIndex, isGLValue)
-    )
-  }
+  override IRType getType() { result = this.getOperand().getIRType() }
 
   final override Location getLocationImpl() { result = this.getOperand().getLocation() }
 
@@ -672,25 +624,6 @@ class IndirectOperand extends Node, TIndirectOperand {
   }
 }
 
-/**
- * The value of an uninitialized local variable, viewed as a node in a data
- * flow graph.
- */
-class UninitializedNode extends Node {
-  LocalVariable v;
-
-  UninitializedNode() {
-    exists(Ssa::Def def |
-      def.getDefiningInstruction() instanceof UninitializedInstruction and
-      Ssa::nodeToDefOrUse(this, def) and
-      v = def.getSourceVariable().getBaseVariable().(Ssa::BaseIRVariable).getIRVariable().getAst()
-    )
-  }
-
-  /** Gets the uninitialized local variable corresponding to this node. */
-  LocalVariable getLocalVariable() { result = v }
-}
-
 /**
  * INTERNAL: Do not use.
  *
@@ -712,11 +645,7 @@ class IndirectInstruction extends Node, TIndirectInstruction {
 
   override Declaration getEnclosingCallable() { result = this.getFunction() }
 
-  override DataFlowType getType() {
-    exists(boolean isGLValue | if instr.isGLValue() then isGLValue = true else isGLValue = false |
-      result = getTypeImpl(instr.getResultType().getUnspecifiedType(), indirectionIndex, isGLValue)
-    )
-  }
+  override IRType getType() { result = this.getInstruction().getResultIRType() }
 
   final override Location getLocationImpl() { result = this.getInstruction().getLocation() }
 
@@ -746,7 +675,7 @@ predicate exprNodeShouldBeOperand(Node node, Expr e) {
 
 /**
  * Holds if `load` is a `LoadInstruction` that is the result of evaluating `e`
- * and `node` is an `IndirectOperandNode` that should map `node.asExpr()` to `e`.
+ * and `node` is an `IndirctOperandNode` that should map `node.asExpr()` to `e`.
  *
  * We map `e` to `node.asExpr()` when `node` semantically represents the
  * same value as `load`. A subsequent flow step will flow `node` to
@@ -930,8 +859,6 @@ abstract class PostUpdateNode extends Node {
    * Gets the node before the state update.
    */
   abstract Node getPreUpdateNode();
-
-  final override DataFlowType getType() { result = this.getPreUpdateNode().getType() }
 }
 
 /**
@@ -995,7 +922,7 @@ class VariableNode extends Node, TVariableNode {
     result = v
   }
 
-  override DataFlowType getType() { result = v.getType() }
+  override IRType getType() { result.getCanonicalLanguageType().hasUnspecifiedType(v.getType(), _) }
 
   final override Location getLocationImpl() { result = v.getLocation() }
 
@@ -1148,7 +1075,7 @@ predicate simpleLocalFlowStep(Node nodeFrom, Node nodeTo) {
       store.getDestinationAddressOperand() = address
     )
     or
-    Ssa::outNodeHasAddressAndIndex(nodeFrom, address, indirectionIndex)
+    Ssa::outNodeHasAddressAndIndex(nodeFrom, address, indirectionIndex - 1)
   )
 }
 

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/ModelUtil.qll

@@ -41,7 +41,7 @@ Node callOutput(CallInstruction call, FunctionOutput output) {
   // The side effect of a call on the value pointed to by an argument or qualifier
   exists(int index, int indirectionIndex |
     result.(IndirectArgumentOutNode).getArgumentIndex() = index and
-    result.(IndirectArgumentOutNode).getIndirectionIndex() = indirectionIndex and
+    result.(IndirectArgumentOutNode).getIndirectionIndex() + 1 = indirectionIndex and
     result.(IndirectArgumentOutNode).getCallInstruction() = call and
     output.isParameterDerefOrQualifierObject(index, indirectionIndex)
   )

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/PrintIRLocalFlow.qll

@@ -100,7 +100,7 @@ private string getNodeProperty(DataFlow::Node node, string key) {
   or
   // Is there partial flow from a source to this node?
   // This property will only be emitted if partial flow is enabled by overriding
-  // `DataFlow::Configuration::explorationLimit()`.
+  // `DataFlow::Configration::explorationLimit()`.
   key = "pflow" and
   result =
     strictconcat(DataFlow::PartialPathNode sourceNode, DataFlow::PartialPathNode destNode, int dist,

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll

@@ -11,9 +11,7 @@ private import DataFlowUtil
  * corresponding `(Indirect)OperandNode`.
  */
 predicate ignoreOperand(Operand operand) {
-  operand = any(Instruction instr | ignoreInstruction(instr)).getAnOperand() or
-  operand = any(Instruction instr | ignoreInstruction(instr)).getAUse() or
-  operand instanceof MemoryOperand
+  operand = any(Instruction instr | ignoreInstruction(instr)).getAnOperand()
 }
 
 /**

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/ssa0/SsaInternals.qll

@@ -36,7 +36,7 @@ private module SourceVariables {
 
     override string toString() { result = var.toString() }
 
-    override DataFlowType getType() { result = var.getType() }
+    override DataFlowType getType() { result = var.getIRType() }
   }
 
   class BaseCallVariable extends BaseSourceVariable, TBaseCallVariable {
@@ -48,7 +48,7 @@ private module SourceVariables {
 
     override string toString() { result = call.toString() }
 
-    override DataFlowType getType() { result = call.getResultType() }
+    override DataFlowType getType() { result = call.getResultIRType() }
   }
 
   private newtype TSourceVariable =

cpp/ql/lib/experimental/semmle/code/cpp/rangeanalysis/ArrayLengthAnalysis.qll

@@ -218,7 +218,7 @@ private predicate allocation(Instruction array, Length length, int delta) {
           length.(VNLength).getInstruction().getConvertedResultExpression() = lengthExpr
         )
         or
-        not deconstructMallocSizeExpr(alloc.getSizeExpr(), _, _) and
+        not exists(int d | deconstructMallocSizeExpr(alloc.getSizeExpr(), _, d)) and
         length.(VNLength).getInstruction().getConvertedResultExpression() = alloc.getSizeExpr() and
         delta = 0
       )

cpp/ql/lib/experimental/semmle/code/cpp/rangeanalysis/RangeAnalysis.qll

@@ -543,7 +543,9 @@ private predicate boundedPhiCand(
   PhiInstruction phi, boolean upper, Bound b, int delta, boolean fromBackEdge, int origdelta,
   Reason reason
 ) {
-  boundedPhiInp(phi, _, b, delta, upper, fromBackEdge, origdelta, reason)
+  exists(PhiInputOperand op |
+    boundedPhiInp(phi, op, b, delta, upper, fromBackEdge, origdelta, reason)
+  )
 }
 
 /**

cpp/ql/lib/experimental/semmle/code/cpp/semantic/SemanticExprSpecific.qll

@@ -292,8 +292,12 @@ module SemanticExprConfig {
     final Location getLocation() { result = super.getLocation() }
   }
 
-  private class ValueNumberBound extends Bound instanceof IRBound::ValueNumberBound {
-    override string toString() { result = IRBound::ValueNumberBound.super.toString() }
+  private class ValueNumberBound extends Bound {
+    IRBound::ValueNumberBound bound;
+
+    ValueNumberBound() { bound = this }
+
+    override string toString() { result = bound.toString() }
   }
 
   predicate zeroBound(Bound bound) { bound instanceof IRBound::ZeroBound }

cpp/ql/lib/experimental/semmle/code/cpp/semantic/analysis/FloatDelta.qll

@@ -1,29 +0,0 @@
-private import RangeAnalysisStage
-
-module FloatDelta implements DeltaSig {
-  class Delta = float;
-
-  bindingset[d]
-  bindingset[result]
-  float toFloat(Delta d) { result = d }
-
-  bindingset[d]
-  bindingset[result]
-  int toInt(Delta d) { result = d }
-
-  bindingset[n]
-  bindingset[result]
-  Delta fromInt(int n) { result = n }
-
-  bindingset[f]
-  Delta fromFloat(float f) {
-    result =
-      min(float diff, float res |
-        diff = (res - f) and res = f.ceil()
-        or
-        diff = (f - res) and res = f.floor()
-      |
-        res order by diff
-      )
-  }
-}

cpp/ql/lib/experimental/semmle/code/cpp/semantic/analysis/ModulusAnalysis.qll

@@ -4,338 +4,308 @@
  * variable), and `v` is an integer in the range `[0 .. m-1]`.
  */
 
-/*
- * The main recursion has base cases in both `ssaModulus` (for guarded reads) and `semExprModulus`
- * (for constant values). The most interesting recursive case is `phiModulusRankStep`, which
- * handles phi inputs.
- */
-
 private import ModulusAnalysisSpecific::Private
 private import experimental.semmle.code.cpp.semantic.Semantic
 private import ConstantAnalysis
 private import RangeUtils
-private import RangeAnalysisStage
 
-module ModulusAnalysis<DeltaSig D, BoundSig<D> Bounds, UtilSig<D> U> {
-  /**
-   * Holds if `e + delta` equals `v` at `pos`.
-   */
-  private predicate valueFlowStepSsa(SemSsaVariable v, SemSsaReadPosition pos, SemExpr e, int delta) {
-    U::semSsaUpdateStep(v, e, D::fromInt(delta)) and pos.hasReadOfVar(v)
-    or
-    exists(SemGuard guard, boolean testIsTrue |
-      pos.hasReadOfVar(v) and
-      guard = U::semEqFlowCond(v, e, D::fromInt(delta), true, testIsTrue) and
-      semGuardDirectlyControlsSsaRead(guard, pos, testIsTrue)
-    )
-  }
+/**
+ * Holds if `e + delta` equals `v` at `pos`.
+ */
+private predicate valueFlowStepSsa(SemSsaVariable v, SemSsaReadPosition pos, SemExpr e, int delta) {
+  semSsaUpdateStep(v, e, delta) and pos.hasReadOfVar(v)
+  or
+  exists(SemGuard guard, boolean testIsTrue |
+    pos.hasReadOfVar(v) and
+    guard = semEqFlowCond(v, e, delta, true, testIsTrue) and
+    semGuardDirectlyControlsSsaRead(guard, pos, testIsTrue)
+  )
+}
 
-  /**
-   * Holds if `add` is the addition of `larg` and `rarg`, neither of which are
-   * `ConstantIntegerExpr`s.
-   */
-  private predicate nonConstAddition(SemExpr add, SemExpr larg, SemExpr rarg) {
-    exists(SemAddExpr a | a = add |
-      larg = a.getLeftOperand() and
-      rarg = a.getRightOperand()
-    ) and
-    not larg instanceof SemConstantIntegerExpr and
-    not rarg instanceof SemConstantIntegerExpr
-  }
+/**
+ * Holds if `add` is the addition of `larg` and `rarg`, neither of which are
+ * `ConstantIntegerExpr`s.
+ */
+private predicate nonConstAddition(SemExpr add, SemExpr larg, SemExpr rarg) {
+  exists(SemAddExpr a | a = add |
+    larg = a.getLeftOperand() and
+    rarg = a.getRightOperand()
+  ) and
+  not larg instanceof SemConstantIntegerExpr and
+  not rarg instanceof SemConstantIntegerExpr
+}
 
-  /**
-   * Holds if `sub` is the subtraction of `larg` and `rarg`, where `rarg` is not
-   * a `ConstantIntegerExpr`.
-   */
-  private predicate nonConstSubtraction(SemExpr sub, SemExpr larg, SemExpr rarg) {
-    exists(SemSubExpr s | s = sub |
-      larg = s.getLeftOperand() and
-      rarg = s.getRightOperand()
-    ) and
-    not rarg instanceof SemConstantIntegerExpr
-  }
+/**
+ * Holds if `sub` is the subtraction of `larg` and `rarg`, where `rarg` is not
+ * a `ConstantIntegerExpr`.
+ */
+private predicate nonConstSubtraction(SemExpr sub, SemExpr larg, SemExpr rarg) {
+  exists(SemSubExpr s | s = sub |
+    larg = s.getLeftOperand() and
+    rarg = s.getRightOperand()
+  ) and
+  not rarg instanceof SemConstantIntegerExpr
+}
 
-  /** Gets an expression that is the remainder modulo `mod` of `arg`. */
-  private SemExpr modExpr(SemExpr arg, int mod) {
-    exists(SemRemExpr rem |
-      result = rem and
-      arg = rem.getLeftOperand() and
-      rem.getRightOperand().(SemConstantIntegerExpr).getIntValue() = mod and
-      mod >= 2
-    )
-    or
-    exists(SemConstantIntegerExpr c |
-      mod = 2.pow([1 .. 30]) and
-      c.getIntValue() = mod - 1 and
-      result.(SemBitAndExpr).hasOperands(arg, c)
-    )
-  }
+/** Gets an expression that is the remainder modulo `mod` of `arg`. */
+private SemExpr modExpr(SemExpr arg, int mod) {
+  exists(SemRemExpr rem |
+    result = rem and
+    arg = rem.getLeftOperand() and
+    rem.getRightOperand().(SemConstantIntegerExpr).getIntValue() = mod and
+    mod >= 2
+  )
+  or
+  exists(SemConstantIntegerExpr c |
+    mod = 2.pow([1 .. 30]) and
+    c.getIntValue() = mod - 1 and
+    result.(SemBitAndExpr).hasOperands(arg, c)
+  )
+}
 
-  /**
-   * Gets a guard that tests whether `v` is congruent with `val` modulo `mod` on
-   * its `testIsTrue` branch.
-   */
-  private SemGuard moduloCheck(SemSsaVariable v, int val, int mod, boolean testIsTrue) {
-    exists(SemExpr rem, SemConstantIntegerExpr c, int r, boolean polarity |
-      result.isEquality(rem, c, polarity) and
-      c.getIntValue() = r and
-      rem = modExpr(v.getAUse(), mod) and
-      (
-        testIsTrue = polarity and val = r
-        or
-        testIsTrue = polarity.booleanNot() and
-        mod = 2 and
-        val = 1 - r and
-        (r = 0 or r = 1)
-      )
-    )
-  }
-
-  /**
-   * Holds if a guard ensures that `v` at `pos` is congruent with `val` modulo `mod`.
-   */
-  private predicate moduloGuardedRead(SemSsaVariable v, SemSsaReadPosition pos, int val, int mod) {
-    exists(SemGuard guard, boolean testIsTrue |
-      pos.hasReadOfVar(v) and
-      guard = moduloCheck(v, val, mod, testIsTrue) and
-      semGuardControlsSsaRead(guard, pos, testIsTrue)
-    )
-  }
-
-  /** Holds if `factor` is a power of 2 that divides `mask`. */
-  bindingset[mask]
-  private predicate andmaskFactor(int mask, int factor) {
-    mask % factor = 0 and
-    factor = 2.pow([1 .. 30])
-  }
-
-  /** Holds if `e` is evenly divisible by `factor`. */
-  private predicate evenlyDivisibleExpr(SemExpr e, int factor) {
-    exists(SemConstantIntegerExpr c, int k | k = c.getIntValue() |
-      e.(SemMulExpr).getAnOperand() = c and factor = k.abs() and factor >= 2
+/**
+ * Gets a guard that tests whether `v` is congruent with `val` modulo `mod` on
+ * its `testIsTrue` branch.
+ */
+private SemGuard moduloCheck(SemSsaVariable v, int val, int mod, boolean testIsTrue) {
+  exists(SemExpr rem, SemConstantIntegerExpr c, int r, boolean polarity |
+    result.isEquality(rem, c, polarity) and
+    c.getIntValue() = r and
+    rem = modExpr(v.getAUse(), mod) and
+    (
+      testIsTrue = polarity and val = r
       or
-      e.(SemShiftLeftExpr).getRightOperand() = c and factor = 2.pow(k) and k > 0
-      or
-      e.(SemBitAndExpr).getAnOperand() = c and factor = max(int f | andmaskFactor(k, f))
+      testIsTrue = polarity.booleanNot() and
+      mod = 2 and
+      val = 1 - r and
+      (r = 0 or r = 1)
     )
-  }
+  )
+}
 
-  /**
-   * Holds if `rix` is the number of input edges to `phi`.
-   */
-  private predicate maxPhiInputRank(SemSsaPhiNode phi, int rix) {
-    rix = max(int r | rankedPhiInput(phi, _, _, r))
-  }
+/**
+ * Holds if a guard ensures that `v` at `pos` is congruent with `val` modulo `mod`.
+ */
+private predicate moduloGuardedRead(SemSsaVariable v, SemSsaReadPosition pos, int val, int mod) {
+  exists(SemGuard guard, boolean testIsTrue |
+    pos.hasReadOfVar(v) and
+    guard = moduloCheck(v, val, mod, testIsTrue) and
+    semGuardControlsSsaRead(guard, pos, testIsTrue)
+  )
+}
 
-  /**
-   * Gets the remainder of `val` modulo `mod`.
-   *
-   * For `mod = 0` the result equals `val` and for `mod > 1` the result is within
-   * the range `[0 .. mod-1]`.
-   */
-  bindingset[val, mod]
-  private int remainder(int val, int mod) {
-    mod = 0 and result = val
+/** Holds if `factor` is a power of 2 that divides `mask`. */
+bindingset[mask]
+private predicate andmaskFactor(int mask, int factor) {
+  mask % factor = 0 and
+  factor = 2.pow([1 .. 30])
+}
+
+/** Holds if `e` is evenly divisible by `factor`. */
+private predicate evenlyDivisibleExpr(SemExpr e, int factor) {
+  exists(SemConstantIntegerExpr c, int k | k = c.getIntValue() |
+    e.(SemMulExpr).getAnOperand() = c and factor = k.abs() and factor >= 2
     or
-    mod > 1 and result = ((val % mod) + mod) % mod
-  }
-
-  /**
-   * Holds if `inp` is an input to `phi` and equals `phi` modulo `mod` along `edge`.
-   */
-  private predicate phiSelfModulus(
-    SemSsaPhiNode phi, SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge, int mod
-  ) {
-    exists(Bounds::SemSsaBound phibound, int v, int m |
-      edge.phiInput(phi, inp) and
-      phibound.getAVariable() = phi and
-      ssaModulus(inp, edge, phibound, v, m) and
-      mod = m.gcd(v) and
-      mod != 1
-    )
-  }
-
-  /**
-   * Holds if `b + val` modulo `mod` is a candidate congruence class for `phi`.
-   */
-  private predicate phiModulusInit(SemSsaPhiNode phi, Bounds::SemBound b, int val, int mod) {
-    exists(SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge |
-      edge.phiInput(phi, inp) and
-      ssaModulus(inp, edge, b, val, mod)
-    )
-  }
-
-  /**
-   * Holds if all inputs to `phi` numbered `1` to `rix` are equal to `b + val` modulo `mod`.
-   */
-  pragma[nomagic]
-  private predicate phiModulusRankStep(
-    SemSsaPhiNode phi, Bounds::SemBound b, int val, int mod, int rix
-  ) {
-    /*
-     * base case. If any phi input is equal to `b + val` modulo `mod`, that's a potential congruence
-     * class for the phi node.
-     */
-
-    rix = 0 and
-    phiModulusInit(phi, b, val, mod)
+    e.(SemShiftLeftExpr).getRightOperand() = c and factor = 2.pow(k) and k > 0
     or
-    exists(SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge, int v1, int m1 |
-      mod != 1 and
-      val = remainder(v1, mod)
-    |
-      /*
-       * Recursive case. If `inp` = `b + v2` mod `m2`, we combine that with the preceding potential
-       * congruence class `b + v1` mod `m1`. The result will be the congruence class of `v1` modulo
-       * the greatest common denominator of `m1`, `m2`, and `v1 - v2`.
-       */
+    e.(SemBitAndExpr).getAnOperand() = c and factor = max(int f | andmaskFactor(k, f))
+  )
+}
 
-      exists(int v2, int m2 |
-        rankedPhiInput(pragma[only_bind_out](phi), inp, edge, rix) and
-        phiModulusRankStep(phi, b, v1, m1, rix - 1) and
-        ssaModulus(inp, edge, b, v2, m2) and
-        mod = m1.gcd(m2).gcd(v1 - v2)
-      )
-      or
-      /*
-       * Recursive case. If `inp` = `phi` mod `m2`, we combine that with the preceding potential
-       * congruence class `b + v1` mod `m1`. The result will be a congruence class modulo the greatest
-       * common denominator of `m1` and `m2`.
-       */
+/**
+ * Holds if `rix` is the number of input edges to `phi`.
+ */
+private predicate maxPhiInputRank(SemSsaPhiNode phi, int rix) {
+  rix = max(int r | rankedPhiInput(phi, _, _, r))
+}
 
-      exists(int m2 |
-        rankedPhiInput(phi, inp, edge, rix) and
-        phiModulusRankStep(phi, b, v1, m1, rix - 1) and
-        phiSelfModulus(phi, inp, edge, m2) and
-        mod = m1.gcd(m2)
-      )
+/**
+ * Gets the remainder of `val` modulo `mod`.
+ *
+ * For `mod = 0` the result equals `val` and for `mod > 1` the result is within
+ * the range `[0 .. mod-1]`.
+ */
+bindingset[val, mod]
+private int remainder(int val, int mod) {
+  mod = 0 and result = val
+  or
+  mod > 1 and result = ((val % mod) + mod) % mod
+}
+
+/**
+ * Holds if `inp` is an input to `phi` and equals `phi` modulo `mod` along `edge`.
+ */
+private predicate phiSelfModulus(
+  SemSsaPhiNode phi, SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge, int mod
+) {
+  exists(SemSsaBound phibound, int v, int m |
+    edge.phiInput(phi, inp) and
+    phibound.getAVariable() = phi and
+    ssaModulus(inp, edge, phibound, v, m) and
+    mod = m.gcd(v) and
+    mod != 1
+  )
+}
+
+/**
+ * Holds if `b + val` modulo `mod` is a candidate congruence class for `phi`.
+ */
+private predicate phiModulusInit(SemSsaPhiNode phi, SemBound b, int val, int mod) {
+  exists(SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge |
+    edge.phiInput(phi, inp) and
+    ssaModulus(inp, edge, b, val, mod)
+  )
+}
+
+/**
+ * Holds if all inputs to `phi` numbered `1` to `rix` are equal to `b + val` modulo `mod`.
+ */
+pragma[nomagic]
+private predicate phiModulusRankStep(SemSsaPhiNode phi, SemBound b, int val, int mod, int rix) {
+  rix = 0 and
+  phiModulusInit(phi, b, val, mod)
+  or
+  exists(SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge, int v1, int m1 |
+    mod != 1 and
+    val = remainder(v1, mod)
+  |
+    exists(int v2, int m2 |
+      rankedPhiInput(pragma[only_bind_out](phi), inp, edge, rix) and
+      phiModulusRankStep(phi, b, v1, m1, rix - 1) and
+      ssaModulus(inp, edge, b, v2, m2) and
+      mod = m1.gcd(m2).gcd(v1 - v2)
     )
-  }
-
-  /**
-   * Holds if `phi` is equal to `b + val` modulo `mod`.
-   */
-  private predicate phiModulus(SemSsaPhiNode phi, Bounds::SemBound b, int val, int mod) {
-    exists(int r |
-      maxPhiInputRank(phi, r) and
-      phiModulusRankStep(phi, b, val, mod, r)
+    or
+    exists(int m2 |
+      rankedPhiInput(phi, inp, edge, rix) and
+      phiModulusRankStep(phi, b, v1, m1, rix - 1) and
+      phiSelfModulus(phi, inp, edge, m2) and
+      mod = m1.gcd(m2)
     )
-  }
+  )
+}
 
-  /**
-   * Holds if `v` at `pos` is equal to `b + val` modulo `mod`.
-   */
-  private predicate ssaModulus(
-    SemSsaVariable v, SemSsaReadPosition pos, Bounds::SemBound b, int val, int mod
-  ) {
-    phiModulus(v, b, val, mod) and pos.hasReadOfVar(v)
+/**
+ * Holds if `phi` is equal to `b + val` modulo `mod`.
+ */
+private predicate phiModulus(SemSsaPhiNode phi, SemBound b, int val, int mod) {
+  exists(int r |
+    maxPhiInputRank(phi, r) and
+    phiModulusRankStep(phi, b, val, mod, r)
+  )
+}
+
+/**
+ * Holds if `v` at `pos` is equal to `b + val` modulo `mod`.
+ */
+private predicate ssaModulus(SemSsaVariable v, SemSsaReadPosition pos, SemBound b, int val, int mod) {
+  phiModulus(v, b, val, mod) and pos.hasReadOfVar(v)
+  or
+  b.(SemSsaBound).getAVariable() = v and pos.hasReadOfVar(v) and val = 0 and mod = 0
+  or
+  exists(SemExpr e, int val0, int delta |
+    semExprModulus(e, b, val0, mod) and
+    valueFlowStepSsa(v, pos, e, delta) and
+    val = remainder(val0 + delta, mod)
+  )
+  or
+  moduloGuardedRead(v, pos, val, mod) and b instanceof SemZeroBound
+}
+
+/**
+ * Holds if `e` is equal to `b + val` modulo `mod`.
+ *
+ * There are two cases for the modulus:
+ * - `mod = 0`: The equality `e = b + val` is an ordinary equality.
+ * - `mod > 1`: `val` lies within the range `[0 .. mod-1]`.
+ */
+cached
+predicate semExprModulus(SemExpr e, SemBound b, int val, int mod) {
+  not ignoreExprModulus(e) and
+  (
+    e = b.getExpr(val) and mod = 0
     or
-    b.(Bounds::SemSsaBound).getAVariable() = v and pos.hasReadOfVar(v) and val = 0 and mod = 0
+    evenlyDivisibleExpr(e, mod) and
+    val = 0 and
+    b instanceof SemZeroBound
     or
-    exists(SemExpr e, int val0, int delta |
-      semExprModulus(e, b, val0, mod) and
-      valueFlowStepSsa(v, pos, e, delta) and
+    exists(SemSsaVariable v, SemSsaReadPositionBlock bb |
+      ssaModulus(v, bb, b, val, mod) and
+      e = v.getAUse() and
+      bb.getAnExpr() = e
+    )
+    or
+    exists(SemExpr mid, int val0, int delta |
+      semExprModulus(mid, b, val0, mod) and
+      semValueFlowStep(e, mid, delta) and
       val = remainder(val0 + delta, mod)
     )
     or
-    moduloGuardedRead(v, pos, val, mod) and b instanceof Bounds::SemZeroBound
-  }
-
-  /**
-   * Holds if `e` is equal to `b + val` modulo `mod`.
-   *
-   * There are two cases for the modulus:
-   * - `mod = 0`: The equality `e = b + val` is an ordinary equality.
-   * - `mod > 1`: `val` lies within the range `[0 .. mod-1]`.
-   */
-  cached
-  predicate semExprModulus(SemExpr e, Bounds::SemBound b, int val, int mod) {
-    not ignoreExprModulus(e) and
-    (
-      e = b.getExpr(D::fromInt(val)) and mod = 0
-      or
-      evenlyDivisibleExpr(e, mod) and
-      val = 0 and
-      b instanceof Bounds::SemZeroBound
-      or
-      exists(SemSsaVariable v, SemSsaReadPositionBlock bb |
-        ssaModulus(v, bb, b, val, mod) and
-        e = v.getAUse() and
-        bb.getAnExpr() = e
-      )
-      or
-      exists(SemExpr mid, int val0, int delta |
-        semExprModulus(mid, b, val0, mod) and
-        U::semValueFlowStep(e, mid, D::fromInt(delta)) and
-        val = remainder(val0 + delta, mod)
-      )
-      or
-      exists(SemConditionalExpr cond, int v1, int v2, int m1, int m2 |
-        cond = e and
-        condExprBranchModulus(cond, true, b, v1, m1) and
-        condExprBranchModulus(cond, false, b, v2, m2) and
-        mod = m1.gcd(m2).gcd(v1 - v2) and
-        mod != 1 and
-        val = remainder(v1, mod)
-      )
-      or
-      exists(Bounds::SemBound b1, Bounds::SemBound b2, int v1, int v2, int m1, int m2 |
-        addModulus(e, true, b1, v1, m1) and
-        addModulus(e, false, b2, v2, m2) and
-        mod = m1.gcd(m2) and
-        mod != 1 and
-        val = remainder(v1 + v2, mod)
-      |
-        b = b1 and b2 instanceof Bounds::SemZeroBound
-        or
-        b = b2 and b1 instanceof Bounds::SemZeroBound
-      )
-      or
-      exists(int v1, int v2, int m1, int m2 |
-        subModulus(e, true, b, v1, m1) and
-        subModulus(e, false, any(Bounds::SemZeroBound zb), v2, m2) and
-        mod = m1.gcd(m2) and
-        mod != 1 and
-        val = remainder(v1 - v2, mod)
-      )
+    exists(SemConditionalExpr cond, int v1, int v2, int m1, int m2 |
+      cond = e and
+      condExprBranchModulus(cond, true, b, v1, m1) and
+      condExprBranchModulus(cond, false, b, v2, m2) and
+      mod = m1.gcd(m2).gcd(v1 - v2) and
+      mod != 1 and
+      val = remainder(v1, mod)
     )
-  }
-
-  private predicate condExprBranchModulus(
-    SemConditionalExpr cond, boolean branch, Bounds::SemBound b, int val, int mod
-  ) {
-    semExprModulus(cond.getBranchExpr(branch), b, val, mod)
-  }
-
-  private predicate addModulus(SemExpr add, boolean isLeft, Bounds::SemBound b, int val, int mod) {
-    exists(SemExpr larg, SemExpr rarg | nonConstAddition(add, larg, rarg) |
-      semExprModulus(larg, b, val, mod) and isLeft = true
+    or
+    exists(SemBound b1, SemBound b2, int v1, int v2, int m1, int m2 |
+      addModulus(e, true, b1, v1, m1) and
+      addModulus(e, false, b2, v2, m2) and
+      mod = m1.gcd(m2) and
+      mod != 1 and
+      val = remainder(v1 + v2, mod)
+    |
+      b = b1 and b2 instanceof SemZeroBound
       or
-      semExprModulus(rarg, b, val, mod) and isLeft = false
+      b = b2 and b1 instanceof SemZeroBound
     )
-  }
-
-  private predicate subModulus(SemExpr sub, boolean isLeft, Bounds::SemBound b, int val, int mod) {
-    exists(SemExpr larg, SemExpr rarg | nonConstSubtraction(sub, larg, rarg) |
-      semExprModulus(larg, b, val, mod) and isLeft = true
-      or
-      semExprModulus(rarg, b, val, mod) and isLeft = false
+    or
+    exists(int v1, int v2, int m1, int m2 |
+      subModulus(e, true, b, v1, m1) and
+      subModulus(e, false, any(SemZeroBound zb), v2, m2) and
+      mod = m1.gcd(m2) and
+      mod != 1 and
+      val = remainder(v1 - v2, mod)
+    )
+  )
+}
+
+private predicate condExprBranchModulus(
+  SemConditionalExpr cond, boolean branch, SemBound b, int val, int mod
+) {
+  semExprModulus(cond.getBranchExpr(branch), b, val, mod)
+}
+
+private predicate addModulus(SemExpr add, boolean isLeft, SemBound b, int val, int mod) {
+  exists(SemExpr larg, SemExpr rarg | nonConstAddition(add, larg, rarg) |
+    semExprModulus(larg, b, val, mod) and isLeft = true
+    or
+    semExprModulus(rarg, b, val, mod) and isLeft = false
+  )
+}
+
+private predicate subModulus(SemExpr sub, boolean isLeft, SemBound b, int val, int mod) {
+  exists(SemExpr larg, SemExpr rarg | nonConstSubtraction(sub, larg, rarg) |
+    semExprModulus(larg, b, val, mod) and isLeft = true
+    or
+    semExprModulus(rarg, b, val, mod) and isLeft = false
+  )
+}
+
+/**
+ * Holds if `inp` is an input to `phi` along `edge` and this input has index `r`
+ * in an arbitrary 1-based numbering of the input edges to `phi`.
+ */
+private predicate rankedPhiInput(
+  SemSsaPhiNode phi, SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge, int r
+) {
+  edge.phiInput(phi, inp) and
+  edge =
+    rank[r](SemSsaReadPositionPhiInputEdge e |
+      e.phiInput(phi, _)
+    |
+      e order by e.getOrigBlock().getUniqueId()
     )
-  }
-
-  /**
-   * Holds if `inp` is an input to `phi` along `edge` and this input has index `r`
-   * in an arbitrary 1-based numbering of the input edges to `phi`.
-   */
-  private predicate rankedPhiInput(
-    SemSsaPhiNode phi, SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge, int r
-  ) {
-    edge.phiInput(phi, inp) and
-    edge =
-      rank[r](SemSsaReadPositionPhiInputEdge e |
-        e.phiInput(phi, _)
-      |
-        e order by e.getOrigBlock().getUniqueId()
-      )
-  }
 }

cpp/ql/lib/experimental/semmle/code/cpp/semantic/analysis/RangeAnalysis.qll

@@ -1,24 +1,832 @@
-private import RangeAnalysisStage
-private import RangeAnalysisSpecific
-private import experimental.semmle.code.cpp.semantic.analysis.FloatDelta
+/**
+ * Provides classes and predicates for range analysis.
+ *
+ * An inferred bound can either be a specific integer, the abstract value of an
+ * SSA variable, or the abstract value of an interesting expression. The latter
+ * category includes array lengths that are not SSA variables.
+ *
+ * If an inferred bound relies directly on a condition, then this condition is
+ * reported as the reason for the bound.
+ */
+
+/*
+ * This library tackles range analysis as a flow problem. Consider e.g.:
+ * ```
+ *   len = arr.length;
+ *   if (x < len) { ... y = x-1; ... y ... }
+ * ```
+ * In this case we would like to infer `y <= arr.length - 2`, and this is
+ * accomplished by tracking the bound through a sequence of steps:
+ * ```
+ *   arr.length --> len = .. --> x < len --> x-1 --> y = .. --> y
+ * ```
+ *
+ * In its simplest form the step relation `E1 --> E2` relates two expressions
+ * such that `E1 <= B` implies `E2 <= B` for any `B` (with a second separate
+ * step relation handling lower bounds). Examples of such steps include
+ * assignments `E2 = E1` and conditions `x <= E1` where `E2` is a use of `x`
+ * guarded by the condition.
+ *
+ * In order to handle subtractions and additions with constants, and strict
+ * comparisons, the step relation is augmented with an integer delta. With this
+ * generalization `E1 --(delta)--> E2` relates two expressions and an integer
+ * such that `E1 <= B` implies `E2 <= B + delta` for any `B`. This corresponds
+ * to the predicate `boundFlowStep`.
+ *
+ * The complete range analysis is then implemented as the transitive closure of
+ * the step relation summing the deltas along the way. If `E1` transitively
+ * steps to `E2`, `delta` is the sum of deltas along the path, and `B` is an
+ * interesting bound equal to the value of `E1` then `E2 <= B + delta`. This
+ * corresponds to the predicate `bounded`.
+ *
+ * Phi nodes need a little bit of extra handling. Consider `x0 = phi(x1, x2)`.
+ * There are essentially two cases:
+ * - If `x1 <= B + d1` and `x2 <= B + d2` then `x0 <= B + max(d1,d2)`.
+ * - If `x1 <= B + d1` and `x2 <= x0 + d2` with `d2 <= 0` then `x0 <= B + d1`.
+ * The first case is for whenever a bound can be proven without taking looping
+ * into account. The second case is relevant when `x2` comes from a back-edge
+ * where we can prove that the variable has been non-increasing through the
+ * loop-iteration as this means that any upper bound that holds prior to the
+ * loop also holds for the variable during the loop.
+ * This generalizes to a phi node with `n` inputs, so if
+ * `x0 = phi(x1, ..., xn)` and `xi <= B + delta` for one of the inputs, then we
+ * also have `x0 <= B + delta` if we can prove either:
+ * - `xj <= B + d` with `d <= delta` or
+ * - `xj <= x0 + d` with `d <= 0`
+ * for each input `xj`.
+ *
+ * As all inferred bounds can be related directly to a path in the source code
+ * the only source of non-termination is if successive redundant (and thereby
+ * increasingly worse) bounds are calculated along a loop in the source code.
+ * We prevent this by weakening the bound to a small finite set of bounds when
+ * a path follows a second back-edge (we postpone weakening till the second
+ * back-edge as a precise bound might require traversing a loop once).
+ */
+
+private import RangeAnalysisSpecific as Specific
 private import RangeUtils
-private import experimental.semmle.code.cpp.semantic.SemanticBound as SemanticBound
+private import SignAnalysisCommon
+private import ModulusAnalysis
+private import experimental.semmle.code.cpp.semantic.Semantic
+private import ConstantAnalysis
 
-module Bounds implements BoundSig<FloatDelta> {
-  class SemBound instanceof SemanticBound::SemBound {
-    string toString() { result = super.toString() }
-
-    SemExpr getExpr(float delta) { result = super.getExpr(delta) }
+cached
+private module RangeAnalysisCache {
+  cached
+  module RangeAnalysisPublic {
+    /**
+     * Holds if `b + delta` is a valid bound for `e`.
+     * - `upper = true`  : `e <= b + delta`
+     * - `upper = false` : `e >= b + delta`
+     *
+     * The reason for the bound is given by `reason` and may be either a condition
+     * or `NoReason` if the bound was proven directly without the use of a bounding
+     * condition.
+     */
+    cached
+    predicate semBounded(SemExpr e, SemBound b, int delta, boolean upper, SemReason reason) {
+      bounded(e, b, delta, upper, _, _, reason) and
+      bestBound(e, b, delta, upper)
+    }
   }
 
-  class SemZeroBound extends SemBound instanceof SemanticBound::SemZeroBound { }
-
-  class SemSsaBound extends SemBound instanceof SemanticBound::SemSsaBound {
-    SemSsaVariable getAVariable() { result = this.(SemanticBound::SemSsaBound).getAVariable() }
+  /**
+   * Holds if `guard = boundFlowCond(_, _, _, _, _) or guard = eqFlowCond(_, _, _, _, _)`.
+   */
+  cached
+  predicate possibleReason(SemGuard guard) {
+    guard = boundFlowCond(_, _, _, _, _) or guard = semEqFlowCond(_, _, _, _, _)
   }
 }
 
-private module CppRangeAnalysis =
-  RangeStage<FloatDelta, Bounds, CppLangImpl, RangeUtil<FloatDelta, CppLangImpl>>;
+private import RangeAnalysisCache
+import RangeAnalysisPublic
 
-import CppRangeAnalysis
+/**
+ * Holds if `b + delta` is a valid bound for `e` and this is the best such delta.
+ * - `upper = true`  : `e <= b + delta`
+ * - `upper = false` : `e >= b + delta`
+ */
+private predicate bestBound(SemExpr e, SemBound b, int delta, boolean upper) {
+  delta = min(int d | bounded(e, b, d, upper, _, _, _)) and upper = true
+  or
+  delta = max(int d | bounded(e, b, d, upper, _, _, _)) and upper = false
+}
+
+/**
+ * Holds if `comp` corresponds to:
+ * - `upper = true`  : `v <= e + delta` or `v < e + delta`
+ * - `upper = false` : `v >= e + delta` or `v > e + delta`
+ */
+private predicate boundCondition(
+  SemRelationalExpr comp, SemSsaVariable v, SemExpr e, int delta, boolean upper
+) {
+  comp.getLesserOperand() = semSsaRead(v, delta) and e = comp.getGreaterOperand() and upper = true
+  or
+  comp.getGreaterOperand() = semSsaRead(v, delta) and e = comp.getLesserOperand() and upper = false
+  or
+  exists(SemSubExpr sub, SemConstantIntegerExpr c, int d |
+    // (v - d) - e < c
+    comp.getLesserOperand() = sub and
+    comp.getGreaterOperand() = c and
+    sub.getLeftOperand() = semSsaRead(v, d) and
+    sub.getRightOperand() = e and
+    upper = true and
+    delta = d + c.getIntValue()
+    or
+    // (v - d) - e > c
+    comp.getGreaterOperand() = sub and
+    comp.getLesserOperand() = c and
+    sub.getLeftOperand() = semSsaRead(v, d) and
+    sub.getRightOperand() = e and
+    upper = false and
+    delta = d + c.getIntValue()
+    or
+    // e - (v - d) < c
+    comp.getLesserOperand() = sub and
+    comp.getGreaterOperand() = c and
+    sub.getLeftOperand() = e and
+    sub.getRightOperand() = semSsaRead(v, d) and
+    upper = false and
+    delta = d - c.getIntValue()
+    or
+    // e - (v - d) > c
+    comp.getGreaterOperand() = sub and
+    comp.getLesserOperand() = c and
+    sub.getLeftOperand() = e and
+    sub.getRightOperand() = semSsaRead(v, d) and
+    upper = true and
+    delta = d - c.getIntValue()
+  )
+}
+
+/**
+ * Holds if `comp` is a comparison between `x` and `y` for which `y - x` has a
+ * fixed value modulo some `mod > 1`, such that the comparison can be
+ * strengthened by `strengthen` when evaluating to `testIsTrue`.
+ */
+private predicate modulusComparison(SemRelationalExpr comp, boolean testIsTrue, int strengthen) {
+  exists(
+    SemBound b, int v1, int v2, int mod1, int mod2, int mod, boolean resultIsStrict, int d, int k
+  |
+    // If `x <= y` and `x =(mod) b + v1` and `y =(mod) b + v2` then
+    // `0 <= y - x =(mod) v2 - v1`. By choosing `k =(mod) v2 - v1` with
+    // `0 <= k < mod` we get `k <= y - x`. If the resulting comparison is
+    // strict then the strengthening amount is instead `k - 1` modulo `mod`:
+    // `x < y` means `0 <= y - x - 1 =(mod) k - 1` so `k - 1 <= y - x - 1` and
+    // thus `k - 1 < y - x` with `0 <= k - 1 < mod`.
+    semExprModulus(comp.getLesserOperand(), b, v1, mod1) and
+    semExprModulus(comp.getGreaterOperand(), b, v2, mod2) and
+    mod = mod1.gcd(mod2) and
+    mod != 1 and
+    (testIsTrue = true or testIsTrue = false) and
+    (
+      if comp.isStrict()
+      then resultIsStrict = testIsTrue
+      else resultIsStrict = testIsTrue.booleanNot()
+    ) and
+    (
+      resultIsStrict = true and d = 1
+      or
+      resultIsStrict = false and d = 0
+    ) and
+    (
+      testIsTrue = true and k = v2 - v1
+      or
+      testIsTrue = false and k = v1 - v2
+    ) and
+    strengthen = (((k - d) % mod) + mod) % mod
+  )
+}
+
+/**
+ * Gets a condition that tests whether `v` is bounded by `e + delta`.
+ *
+ * If the condition evaluates to `testIsTrue`:
+ * - `upper = true`  : `v <= e + delta`
+ * - `upper = false` : `v >= e + delta`
+ */
+private SemGuard boundFlowCond(
+  SemSsaVariable v, SemExpr e, int delta, boolean upper, boolean testIsTrue
+) {
+  exists(
+    SemRelationalExpr comp, int d1, int d2, int d3, int strengthen, boolean compIsUpper,
+    boolean resultIsStrict
+  |
+    comp = result.asExpr() and
+    boundCondition(comp, v, e, d1, compIsUpper) and
+    (testIsTrue = true or testIsTrue = false) and
+    upper = compIsUpper.booleanXor(testIsTrue.booleanNot()) and
+    (
+      if comp.isStrict()
+      then resultIsStrict = testIsTrue
+      else resultIsStrict = testIsTrue.booleanNot()
+    ) and
+    (
+      if
+        getTrackedTypeForSsaVariable(v) instanceof SemIntegerType or
+        getTrackedTypeForSsaVariable(v) instanceof SemAddressType
+      then
+        upper = true and strengthen = -1
+        or
+        upper = false and strengthen = 1
+      else strengthen = 0
+    ) and
+    (
+      exists(int k | modulusComparison(comp, testIsTrue, k) and d2 = strengthen * k)
+      or
+      not modulusComparison(comp, testIsTrue, _) and d2 = 0
+    ) and
+    // A strict inequality `x < y` can be strengthened to `x <= y - 1`.
+    (
+      resultIsStrict = true and d3 = strengthen
+      or
+      resultIsStrict = false and d3 = 0
+    ) and
+    delta = d1 + d2 + d3
+  )
+  or
+  exists(boolean testIsTrue0 |
+    semImplies_v2(result, testIsTrue, boundFlowCond(v, e, delta, upper, testIsTrue0), testIsTrue0)
+  )
+  or
+  result = semEqFlowCond(v, e, delta, true, testIsTrue) and
+  (upper = true or upper = false)
+  or
+  // guard that tests whether `v2` is bounded by `e + delta + d1 - d2` and
+  // exists a guard `guardEq` such that `v = v2 - d1 + d2`.
+  exists(SemSsaVariable v2, SemGuard guardEq, boolean eqIsTrue, int d1, int d2 |
+    guardEq = semEqFlowCond(v, semSsaRead(v2, d1), d2, true, eqIsTrue) and
+    result = boundFlowCond(v2, e, delta + d1 - d2, upper, testIsTrue) and
+    // guardEq needs to control guard
+    guardEq.directlyControls(result.getBasicBlock(), eqIsTrue)
+  )
+}
+
+private newtype TSemReason =
+  TSemNoReason() or
+  TSemCondReason(SemGuard guard) { possibleReason(guard) }
+
+/**
+ * A reason for an inferred bound. This can either be `CondReason` if the bound
+ * is due to a specific condition, or `NoReason` if the bound is inferred
+ * without going through a bounding condition.
+ */
+abstract class SemReason extends TSemReason {
+  /** Gets a textual representation of this reason. */
+  abstract string toString();
+}
+
+/**
+ * A reason for an inferred bound that indicates that the bound is inferred
+ * without going through a bounding condition.
+ */
+class SemNoReason extends SemReason, TSemNoReason {
+  override string toString() { result = "NoReason" }
+}
+
+/** A reason for an inferred bound pointing to a condition. */
+class SemCondReason extends SemReason, TSemCondReason {
+  /** Gets the condition that is the reason for the bound. */
+  SemGuard getCond() { this = TSemCondReason(result) }
+
+  override string toString() { result = getCond().toString() }
+}
+
+/**
+ * Holds if `e + delta` is a valid bound for `v` at `pos`.
+ * - `upper = true`  : `v <= e + delta`
+ * - `upper = false` : `v >= e + delta`
+ */
+private predicate boundFlowStepSsa(
+  SemSsaVariable v, SemSsaReadPosition pos, SemExpr e, int delta, boolean upper, SemReason reason
+) {
+  semSsaUpdateStep(v, e, delta) and
+  pos.hasReadOfVar(v) and
+  (upper = true or upper = false) and
+  reason = TSemNoReason()
+  or
+  exists(SemGuard guard, boolean testIsTrue |
+    pos.hasReadOfVar(v) and
+    guard = boundFlowCond(v, e, delta, upper, testIsTrue) and
+    semGuardDirectlyControlsSsaRead(guard, pos, testIsTrue) and
+    reason = TSemCondReason(guard)
+  )
+}
+
+/** Holds if `v != e + delta` at `pos` and `v` is of integral type. */
+private predicate unequalFlowStepIntegralSsa(
+  SemSsaVariable v, SemSsaReadPosition pos, SemExpr e, int delta, SemReason reason
+) {
+  getTrackedTypeForSsaVariable(v) instanceof SemIntegerType and
+  exists(SemGuard guard, boolean testIsTrue |
+    pos.hasReadOfVar(v) and
+    guard = semEqFlowCond(v, e, delta, false, testIsTrue) and
+    semGuardDirectlyControlsSsaRead(guard, pos, testIsTrue) and
+    reason = TSemCondReason(guard)
+  )
+}
+
+/**
+ * An expression that does conversion, boxing, or unboxing
+ */
+private class ConvertOrBoxExpr extends SemUnaryExpr {
+  ConvertOrBoxExpr() {
+    this instanceof SemConvertExpr
+    or
+    this instanceof SemBoxExpr
+    or
+    this instanceof SemUnboxExpr
+  }
+}
+
+/**
+ * A cast that can be ignored for the purpose of range analysis.
+ */
+private class SafeCastExpr extends ConvertOrBoxExpr {
+  SafeCastExpr() {
+    conversionCannotOverflow(getTrackedType(pragma[only_bind_into](getOperand())),
+      getTrackedType(this))
+  }
+}
+
+/**
+ * Holds if `typ` is a small integral type with the given lower and upper bounds.
+ */
+private predicate typeBound(SemIntegerType typ, int lowerbound, int upperbound) {
+  exists(int bitSize | bitSize = typ.getByteSize() * 8 |
+    bitSize < 32 and
+    (
+      if typ.isSigned()
+      then (
+        upperbound = 1.bitShiftLeft(bitSize - 1) - 1 and
+        lowerbound = -upperbound - 1
+      ) else (
+        lowerbound = 0 and
+        upperbound = 1.bitShiftLeft(bitSize) - 1
+      )
+    )
+  )
+}
+
+/**
+ * A cast to a small integral type that may overflow or underflow.
+ */
+private class NarrowingCastExpr extends ConvertOrBoxExpr {
+  NarrowingCastExpr() {
+    not this instanceof SafeCastExpr and
+    typeBound(getTrackedType(this), _, _)
+  }
+
+  /** Gets the lower bound of the resulting type. */
+  int getLowerBound() { typeBound(getTrackedType(this), result, _) }
+
+  /** Gets the upper bound of the resulting type. */
+  int getUpperBound() { typeBound(getTrackedType(this), _, result) }
+}
+
+/** Holds if `e >= 1` as determined by sign analysis. */
+private predicate strictlyPositiveIntegralExpr(SemExpr e) {
+  semStrictlyPositive(e) and getTrackedType(e) instanceof SemIntegerType
+}
+
+/** Holds if `e <= -1` as determined by sign analysis. */
+private predicate strictlyNegativeIntegralExpr(SemExpr e) {
+  semStrictlyNegative(e) and getTrackedType(e) instanceof SemIntegerType
+}
+
+/**
+ * Holds if `e1 + delta` is a valid bound for `e2`.
+ * - `upper = true`  : `e2 <= e1 + delta`
+ * - `upper = false` : `e2 >= e1 + delta`
+ */
+private predicate boundFlowStep(SemExpr e2, SemExpr e1, int delta, boolean upper) {
+  semValueFlowStep(e2, e1, delta) and
+  (upper = true or upper = false)
+  or
+  e2.(SafeCastExpr).getOperand() = e1 and
+  delta = 0 and
+  (upper = true or upper = false)
+  or
+  exists(SemExpr x | e2.(SemAddExpr).hasOperands(e1, x) |
+    // `x instanceof ConstantIntegerExpr` is covered by valueFlowStep
+    not x instanceof SemConstantIntegerExpr and
+    not e1 instanceof SemConstantIntegerExpr and
+    if strictlyPositiveIntegralExpr(x)
+    then upper = false and delta = 1
+    else
+      if semPositive(x)
+      then upper = false and delta = 0
+      else
+        if strictlyNegativeIntegralExpr(x)
+        then upper = true and delta = -1
+        else
+          if semNegative(x)
+          then upper = true and delta = 0
+          else none()
+  )
+  or
+  exists(SemExpr x, SemSubExpr sub |
+    e2 = sub and
+    sub.getLeftOperand() = e1 and
+    sub.getRightOperand() = x
+  |
+    // `x instanceof ConstantIntegerExpr` is covered by valueFlowStep
+    not x instanceof SemConstantIntegerExpr and
+    if strictlyPositiveIntegralExpr(x)
+    then upper = true and delta = -1
+    else
+      if semPositive(x)
+      then upper = true and delta = 0
+      else
+        if strictlyNegativeIntegralExpr(x)
+        then upper = false and delta = 1
+        else
+          if semNegative(x)
+          then upper = false and delta = 0
+          else none()
+  )
+  or
+  e2.(SemRemExpr).getRightOperand() = e1 and
+  semPositive(e1) and
+  delta = -1 and
+  upper = true
+  or
+  e2.(SemRemExpr).getLeftOperand() = e1 and semPositive(e1) and delta = 0 and upper = true
+  or
+  e2.(SemBitAndExpr).getAnOperand() = e1 and
+  semPositive(e1) and
+  delta = 0 and
+  upper = true
+  or
+  e2.(SemBitOrExpr).getAnOperand() = e1 and
+  semPositive(e2) and
+  delta = 0 and
+  upper = false
+  or
+  Specific::hasBound(e2, e1, delta, upper)
+}
+
+/** Holds if `e2 = e1 * factor` and `factor > 0`. */
+private predicate boundFlowStepMul(SemExpr e2, SemExpr e1, int factor) {
+  exists(SemConstantIntegerExpr c, int k | k = c.getIntValue() and k > 0 |
+    e2.(SemMulExpr).hasOperands(e1, c) and factor = k
+    or
+    exists(SemShiftLeftExpr e |
+      e = e2 and e.getLeftOperand() = e1 and e.getRightOperand() = c and factor = 2.pow(k)
+    )
+  )
+}
+
+/**
+ * Holds if `e2 = e1 / factor` and `factor > 0`.
+ *
+ * This conflates division, right shift, and unsigned right shift and is
+ * therefore only valid for non-negative numbers.
+ */
+private predicate boundFlowStepDiv(SemExpr e2, SemExpr e1, int factor) {
+  exists(SemConstantIntegerExpr c, int k | k = c.getIntValue() and k > 0 |
+    exists(SemDivExpr e |
+      e = e2 and e.getLeftOperand() = e1 and e.getRightOperand() = c and factor = k
+    )
+    or
+    exists(SemShiftRightExpr e |
+      e = e2 and e.getLeftOperand() = e1 and e.getRightOperand() = c and factor = 2.pow(k)
+    )
+    or
+    exists(SemShiftRightUnsignedExpr e |
+      e = e2 and e.getLeftOperand() = e1 and e.getRightOperand() = c and factor = 2.pow(k)
+    )
+  )
+}
+
+/**
+ * Holds if `b + delta` is a valid bound for `v` at `pos`.
+ * - `upper = true`  : `v <= b + delta`
+ * - `upper = false` : `v >= b + delta`
+ */
+private predicate boundedSsa(
+  SemSsaVariable v, SemSsaReadPosition pos, SemBound b, int delta, boolean upper,
+  boolean fromBackEdge, int origdelta, SemReason reason
+) {
+  exists(SemExpr mid, int d1, int d2, SemReason r1, SemReason r2 |
+    boundFlowStepSsa(v, pos, mid, d1, upper, r1) and
+    bounded(mid, b, d2, upper, fromBackEdge, origdelta, r2) and
+    // upper = true:  v <= mid + d1 <= b + d1 + d2 = b + delta
+    // upper = false: v >= mid + d1 >= b + d1 + d2 = b + delta
+    delta = d1 + d2 and
+    (if r1 instanceof SemNoReason then reason = r2 else reason = r1)
+  )
+  or
+  exists(int d, SemReason r1, SemReason r2 |
+    boundedSsa(v, pos, b, d, upper, fromBackEdge, origdelta, r2) or
+    boundedPhi(v, b, d, upper, fromBackEdge, origdelta, r2)
+  |
+    unequalIntegralSsa(v, pos, b, d, r1) and
+    (
+      upper = true and delta = d - 1
+      or
+      upper = false and delta = d + 1
+    ) and
+    (
+      reason = r1
+      or
+      reason = r2 and not r2 instanceof SemNoReason
+    )
+  )
+}
+
+/**
+ * Holds if `v != b + delta` at `pos` and `v` is of integral type.
+ */
+private predicate unequalIntegralSsa(
+  SemSsaVariable v, SemSsaReadPosition pos, SemBound b, int delta, SemReason reason
+) {
+  exists(SemExpr e, int d1, int d2 |
+    unequalFlowStepIntegralSsa(v, pos, e, d1, reason) and
+    boundedUpper(e, b, d1) and
+    boundedLower(e, b, d2) and
+    delta = d2 + d1
+  )
+}
+
+/**
+ * Holds if `b + delta` is an upper bound for `e`.
+ *
+ * This predicate only exists to prevent a bad standard order in `unequalIntegralSsa`.
+ */
+pragma[nomagic]
+private predicate boundedUpper(SemExpr e, SemBound b, int delta) {
+  bounded(e, b, delta, true, _, _, _)
+}
+
+/**
+ * Holds if `b + delta` is a lower bound for `e`.
+ *
+ * This predicate only exists to prevent a bad standard order in `unequalIntegralSsa`.
+ */
+pragma[nomagic]
+private predicate boundedLower(SemExpr e, SemBound b, int delta) {
+  bounded(e, b, delta, false, _, _, _)
+}
+
+/** Weakens a delta to lie in the range `[-1..1]`. */
+bindingset[delta, upper]
+private int weakenDelta(boolean upper, int delta) {
+  delta in [-1 .. 1] and result = delta
+  or
+  upper = true and result = -1 and delta < -1
+  or
+  upper = false and result = 1 and delta > 1
+}
+
+/**
+ * Holds if `b + delta` is a valid bound for `inp` when used as an input to
+ * `phi` along `edge`.
+ * - `upper = true`  : `inp <= b + delta`
+ * - `upper = false` : `inp >= b + delta`
+ */
+private predicate boundedPhiInp(
+  SemSsaPhiNode phi, SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge, SemBound b, int delta,
+  boolean upper, boolean fromBackEdge, int origdelta, SemReason reason
+) {
+  edge.phiInput(phi, inp) and
+  exists(int d, boolean fromBackEdge0 |
+    boundedSsa(inp, edge, b, d, upper, fromBackEdge0, origdelta, reason)
+    or
+    boundedPhi(inp, b, d, upper, fromBackEdge0, origdelta, reason)
+    or
+    b.(SemSsaBound).getAVariable() = inp and
+    d = 0 and
+    (upper = true or upper = false) and
+    fromBackEdge0 = false and
+    origdelta = 0 and
+    reason = TSemNoReason()
+  |
+    if semBackEdge(phi, inp, edge)
+    then
+      fromBackEdge = true and
+      (
+        fromBackEdge0 = true and delta = weakenDelta(upper, d - origdelta) + origdelta
+        or
+        fromBackEdge0 = false and delta = d
+      )
+    else (
+      delta = d and fromBackEdge = fromBackEdge0
+    )
+  )
+}
+
+/**
+ * Holds if `b + delta` is a valid bound for `inp` when used as an input to
+ * `phi` along `edge`.
+ * - `upper = true`  : `inp <= b + delta`
+ * - `upper = false` : `inp >= b + delta`
+ *
+ * Equivalent to `boundedPhiInp(phi, inp, edge, b, delta, upper, _, _, _)`.
+ */
+pragma[noinline]
+private predicate boundedPhiInp1(
+  SemSsaPhiNode phi, SemBound b, boolean upper, SemSsaVariable inp,
+  SemSsaReadPositionPhiInputEdge edge, int delta
+) {
+  boundedPhiInp(phi, inp, edge, b, delta, upper, _, _, _)
+}
+
+/**
+ * Holds if `phi` is a valid bound for `inp` when used as an input to `phi`
+ * along `edge`.
+ * - `upper = true`  : `inp <= phi`
+ * - `upper = false` : `inp >= phi`
+ */
+private predicate selfBoundedPhiInp(
+  SemSsaPhiNode phi, SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge, boolean upper
+) {
+  exists(int d, SemSsaBound phibound |
+    phibound.getAVariable() = phi and
+    boundedPhiInp(phi, inp, edge, phibound, d, upper, _, _, _) and
+    (
+      upper = true and d <= 0
+      or
+      upper = false and d >= 0
+    )
+  )
+}
+
+/**
+ * Holds if `b + delta` is a valid bound for some input, `inp`, to `phi`, and
+ * thus a candidate bound for `phi`.
+ * - `upper = true`  : `inp <= b + delta`
+ * - `upper = false` : `inp >= b + delta`
+ */
+pragma[noinline]
+private predicate boundedPhiCand(
+  SemSsaPhiNode phi, boolean upper, SemBound b, int delta, boolean fromBackEdge, int origdelta,
+  SemReason reason
+) {
+  exists(SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge |
+    boundedPhiInp(phi, inp, edge, b, delta, upper, fromBackEdge, origdelta, reason)
+  )
+}
+
+/**
+ * Holds if the candidate bound `b + delta` for `phi` is valid for the phi input
+ * `inp` along `edge`.
+ */
+private predicate boundedPhiCandValidForEdge(
+  SemSsaPhiNode phi, SemBound b, int delta, boolean upper, boolean fromBackEdge, int origdelta,
+  SemReason reason, SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge
+) {
+  boundedPhiCand(phi, upper, b, delta, fromBackEdge, origdelta, reason) and
+  (
+    exists(int d | boundedPhiInp1(phi, b, upper, inp, edge, d) | upper = true and d <= delta)
+    or
+    exists(int d | boundedPhiInp1(phi, b, upper, inp, edge, d) | upper = false and d >= delta)
+    or
+    selfBoundedPhiInp(phi, inp, edge, upper)
+  )
+}
+
+/**
+ * Holds if `b + delta` is a valid bound for `phi`.
+ * - `upper = true`  : `phi <= b + delta`
+ * - `upper = false` : `phi >= b + delta`
+ */
+private predicate boundedPhi(
+  SemSsaPhiNode phi, SemBound b, int delta, boolean upper, boolean fromBackEdge, int origdelta,
+  SemReason reason
+) {
+  forex(SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge | edge.phiInput(phi, inp) |
+    boundedPhiCandValidForEdge(phi, b, delta, upper, fromBackEdge, origdelta, reason, inp, edge)
+  )
+}
+
+/**
+ * Holds if `e` has an upper (for `upper = true`) or lower
+ * (for `upper = false`) bound of `b`.
+ */
+private predicate baseBound(SemExpr e, int b, boolean upper) {
+  Specific::hasConstantBound(e, b, upper)
+  or
+  upper = false and
+  b = 0 and
+  semPositive(e.(SemBitAndExpr).getAnOperand()) and
+  // REVIEW: We let the language opt out here to preserve original results.
+  not Specific::ignoreZeroLowerBound(e)
+}
+
+/**
+ * Holds if the value being cast has an upper (for `upper = true`) or lower
+ * (for `upper = false`) bound within the bounds of the resulting type.
+ * For `upper = true` this means that the cast will not overflow and for
+ * `upper = false` this means that the cast will not underflow.
+ */
+private predicate safeNarrowingCast(NarrowingCastExpr cast, boolean upper) {
+  exists(int bound | bounded(cast.getOperand(), any(SemZeroBound zb), bound, upper, _, _, _) |
+    upper = true and bound <= cast.getUpperBound()
+    or
+    upper = false and bound >= cast.getLowerBound()
+  )
+}
+
+pragma[noinline]
+private predicate boundedCastExpr(
+  NarrowingCastExpr cast, SemBound b, int delta, boolean upper, boolean fromBackEdge, int origdelta,
+  SemReason reason
+) {
+  bounded(cast.getOperand(), b, delta, upper, fromBackEdge, origdelta, reason)
+}
+
+/**
+ * Holds if `b + delta` is a valid bound for `e`.
+ * - `upper = true`  : `e <= b + delta`
+ * - `upper = false` : `e >= b + delta`
+ */
+private predicate bounded(
+  SemExpr e, SemBound b, int delta, boolean upper, boolean fromBackEdge, int origdelta,
+  SemReason reason
+) {
+  not Specific::ignoreExprBound(e) and
+  (
+    e = b.getExpr(delta) and
+    (upper = true or upper = false) and
+    fromBackEdge = false and
+    origdelta = delta and
+    reason = TSemNoReason()
+    or
+    baseBound(e, delta, upper) and
+    b instanceof SemZeroBound and
+    fromBackEdge = false and
+    origdelta = delta and
+    reason = TSemNoReason()
+    or
+    exists(SemSsaVariable v, SemSsaReadPositionBlock bb |
+      boundedSsa(v, bb, b, delta, upper, fromBackEdge, origdelta, reason) and
+      e = v.getAUse() and
+      bb.getBlock() = e.getBasicBlock()
+    )
+    or
+    exists(SemExpr mid, int d1, int d2 |
+      boundFlowStep(e, mid, d1, upper) and
+      // Constants have easy, base-case bounds, so let's not infer any recursive bounds.
+      not e instanceof SemConstantIntegerExpr and
+      bounded(mid, b, d2, upper, fromBackEdge, origdelta, reason) and
+      // upper = true:  e <= mid + d1 <= b + d1 + d2 = b + delta
+      // upper = false: e >= mid + d1 >= b + d1 + d2 = b + delta
+      delta = d1 + d2
+    )
+    or
+    exists(SemSsaPhiNode phi |
+      boundedPhi(phi, b, delta, upper, fromBackEdge, origdelta, reason) and
+      e = phi.getAUse()
+    )
+    or
+    exists(SemExpr mid, int factor, int d |
+      boundFlowStepMul(e, mid, factor) and
+      not e instanceof SemConstantIntegerExpr and
+      bounded(mid, b, d, upper, fromBackEdge, origdelta, reason) and
+      b instanceof SemZeroBound and
+      delta = d * factor
+    )
+    or
+    exists(SemExpr mid, int factor, int d |
+      boundFlowStepDiv(e, mid, factor) and
+      not e instanceof SemConstantIntegerExpr and
+      bounded(mid, b, d, upper, fromBackEdge, origdelta, reason) and
+      b instanceof SemZeroBound and
+      d >= 0 and
+      delta = d / factor
+    )
+    or
+    exists(NarrowingCastExpr cast |
+      cast = e and
+      safeNarrowingCast(cast, upper.booleanNot()) and
+      boundedCastExpr(cast, b, delta, upper, fromBackEdge, origdelta, reason)
+    )
+    or
+    exists(
+      SemConditionalExpr cond, int d1, int d2, boolean fbe1, boolean fbe2, int od1, int od2,
+      SemReason r1, SemReason r2
+    |
+      cond = e and
+      boundedConditionalExpr(cond, b, upper, true, d1, fbe1, od1, r1) and
+      boundedConditionalExpr(cond, b, upper, false, d2, fbe2, od2, r2) and
+      (
+        delta = d1 and fromBackEdge = fbe1 and origdelta = od1 and reason = r1
+        or
+        delta = d2 and fromBackEdge = fbe2 and origdelta = od2 and reason = r2
+      )
+    |
+      upper = true and delta = d1.maximum(d2)
+      or
+      upper = false and delta = d1.minimum(d2)
+    )
+  )
+}
+
+private predicate boundedConditionalExpr(
+  SemConditionalExpr cond, SemBound b, boolean upper, boolean branch, int delta,
+  boolean fromBackEdge, int origdelta, SemReason reason
+) {
+  bounded(cond.getBranchExpr(branch), b, delta, upper, fromBackEdge, origdelta, reason)
+}

cpp/ql/lib/experimental/semmle/code/cpp/semantic/analysis/RangeAnalysisSpecific.qll

@@ -3,90 +3,86 @@
  */
 
 private import experimental.semmle.code.cpp.semantic.Semantic
-private import RangeAnalysisStage
-private import experimental.semmle.code.cpp.semantic.analysis.FloatDelta
 
-module CppLangImpl implements LangSig<FloatDelta> {
-  /**
-   * Holds if the specified expression should be excluded from the result of `ssaRead()`.
-   *
-   * This predicate is to keep the results identical to the original Java implementation. It should be
-   * removed once we have the new implementation matching the old results exactly.
-   */
-  predicate ignoreSsaReadCopy(SemExpr e) { none() }
+/**
+ * Holds if the specified expression should be excluded from the result of `ssaRead()`.
+ *
+ * This predicate is to keep the results identical to the original Java implementation. It should be
+ * removed once we have the new implementation matching the old results exactly.
+ */
+predicate ignoreSsaReadCopy(SemExpr e) { none() }
 
-  /**
-   * Ignore the bound on this expression.
-   *
-   * This predicate is to keep the results identical to the original Java implementation. It should be
-   * removed once we have the new implementation matching the old results exactly.
-   */
-  predicate ignoreExprBound(SemExpr e) { none() }
+/**
+ * Ignore the bound on this expression.
+ *
+ * This predicate is to keep the results identical to the original Java implementation. It should be
+ * removed once we have the new implementation matching the old results exactly.
+ */
+predicate ignoreExprBound(SemExpr e) { none() }
 
-  /**
-   * Ignore any inferred zero lower bound on this expression.
-   *
-   * This predicate is to keep the results identical to the original Java implementation. It should be
-   * removed once we have the new implementation matching the old results exactly.
-   */
-  predicate ignoreZeroLowerBound(SemExpr e) { none() }
+/**
+ * Ignore any inferred zero lower bound on this expression.
+ *
+ * This predicate is to keep the results identical to the original Java implementation. It should be
+ * removed once we have the new implementation matching the old results exactly.
+ */
+predicate ignoreZeroLowerBound(SemExpr e) { none() }
 
-  /**
-   * Holds if the specified expression should be excluded from the result of `ssaRead()`.
-   *
-   * This predicate is to keep the results identical to the original Java implementation. It should be
-   * removed once we have the new implementation matching the old results exactly.
-   */
-  predicate ignoreSsaReadArithmeticExpr(SemExpr e) { none() }
+/**
+ * Holds if the specified expression should be excluded from the result of `ssaRead()`.
+ *
+ * This predicate is to keep the results identical to the original Java implementation. It should be
+ * removed once we have the new implementation matching the old results exactly.
+ */
+predicate ignoreSsaReadArithmeticExpr(SemExpr e) { none() }
 
-  /**
-   * Holds if the specified variable should be excluded from the result of `ssaRead()`.
-   *
-   * This predicate is to keep the results identical to the original Java implementation. It should be
-   * removed once we have the new implementation matching the old results exactly.
-   */
-  predicate ignoreSsaReadAssignment(SemSsaVariable v) { none() }
+/**
+ * Holds if the specified variable should be excluded from the result of `ssaRead()`.
+ *
+ * This predicate is to keep the results identical to the original Java implementation. It should be
+ * removed once we have the new implementation matching the old results exactly.
+ */
+predicate ignoreSsaReadAssignment(SemSsaVariable v) { none() }
 
-  /**
-   * Adds additional results to `ssaRead()` that are specific to Java.
-   *
-   * This predicate handles propagation of offsets for post-increment and post-decrement expressions
-   * in exactly the same way as the old Java implementation. Once the new implementation matches the
-   * old one, we should remove this predicate and propagate deltas for all similar patterns, whether
-   * or not they come from a post-increment/decrement expression.
-   */
-  SemExpr specificSsaRead(SemSsaVariable v, float delta) { none() }
+/**
+ * Adds additional results to `ssaRead()` that are specific to Java.
+ *
+ * This predicate handles propagation of offsets for post-increment and post-decrement expressions
+ * in exactly the same way as the old Java implementation. Once the new implementation matches the
+ * old one, we should remove this predicate and propagate deltas for all similar patterns, whether
+ * or not they come from a post-increment/decrement expression.
+ */
+SemExpr specificSsaRead(SemSsaVariable v, int delta) { none() }
 
-  /**
-   * Holds if `e >= bound` (if `upper = false`) or `e <= bound` (if `upper = true`).
-   */
-  predicate hasConstantBound(SemExpr e, float bound, boolean upper) { none() }
+/**
+ * Holds if `e >= bound` (if `upper = false`) or `e <= bound` (if `upper = true`).
+ */
+predicate hasConstantBound(SemExpr e, int bound, boolean upper) { none() }
 
-  /**
-   * Holds if `e >= bound + delta` (if `upper = false`) or `e <= bound + delta` (if `upper = true`).
-   */
-  predicate hasBound(SemExpr e, SemExpr bound, float delta, boolean upper) { none() }
+/**
+ * Holds if `e >= bound + delta` (if `upper = false`) or `e <= bound + delta` (if `upper = true`).
+ */
+predicate hasBound(SemExpr e, SemExpr bound, int delta, boolean upper) { none() }
 
-  /**
-   * Holds if the value of `dest` is known to be `src + delta`.
-   */
-  predicate additionalValueFlowStep(SemExpr dest, SemExpr src, float delta) { none() }
+/**
+ * Holds if the value of `dest` is known to be `src + delta`.
+ */
+predicate additionalValueFlowStep(SemExpr dest, SemExpr src, int delta) { none() }
 
-  /**
-   * Gets the type that range analysis should use to track the result of the specified expression,
-   * if a type other than the original type of the expression is to be used.
-   *
-   * This predicate is commonly used in languages that support immutable "boxed" types that are
-   * actually references but whose values can be tracked as the type contained in the box.
-   */
-  SemType getAlternateType(SemExpr e) { none() }
+/**
+ * Gets the type that range analysis should use to track the result of the specified expression,
+ * if a type other than the original type of the expression is to be used.
+ *
+ * This predicate is commonly used in languages that support immutable "boxed" types that are
+ * actually references but whose values can be tracked as the type contained in the box.
+ */
+SemType getAlternateType(SemExpr e) { none() }
 
-  /**
-   * Gets the type that range analysis should use to track the result of the specified source
-   * variable, if a type other than the original type of the expression is to be used.
-   *
-   * This predicate is commonly used in languages that support immutable "boxed" types that are
-   * actually references but whose values can be tracked as the type contained in the box.
-   */
-  SemType getAlternateTypeForSsaVariable(SemSsaVariable var) { none() }
-}
+/**
+ * Gets the type that range analysis should use to track the result of the specified source
+ * variable, if a type other than the original type of the expression is to be used.
+ *
+ * This predicate is commonly used in languages that support immutable "boxed" types that are
+ * actually references but whose values can be tracked as the type contained in the box.
+ */
+SemType getAlternateTypeForSsaVariable(SemSsaVariable var) { none() }

cpp/ql/lib/experimental/semmle/code/cpp/semantic/analysis/RangeUtils.qll

@@ -3,138 +3,133 @@
  */
 
 private import experimental.semmle.code.cpp.semantic.Semantic
-private import RangeAnalysisSpecific
-private import RangeAnalysisStage as Range
+private import RangeAnalysisSpecific as Specific
 private import ConstantAnalysis
 
-module RangeUtil<Range::DeltaSig D, Range::LangSig<D> Lang> implements Range::UtilSig<D> {
-  /**
-   * Gets an expression that equals `v - d`.
-   */
-  SemExpr semSsaRead(SemSsaVariable v, D::Delta delta) {
-    // There are various language-specific extension points that can be removed once we no longer
-    // expect to match the original Java implementation's results exactly.
-    result = v.getAUse() and delta = D::fromInt(0)
-    or
-    exists(D::Delta d1, SemConstantIntegerExpr c |
-      result.(SemAddExpr).hasOperands(semSsaRead(v, d1), c) and
-      delta = D::fromFloat(D::toFloat(d1) - c.getIntValue()) and
-      not Lang::ignoreSsaReadArithmeticExpr(result)
-    )
-    or
-    exists(SemSubExpr sub, D::Delta d1, SemConstantIntegerExpr c |
-      result = sub and
-      sub.getLeftOperand() = semSsaRead(v, d1) and
-      sub.getRightOperand() = c and
-      delta = D::fromFloat(D::toFloat(d1) + c.getIntValue()) and
-      not Lang::ignoreSsaReadArithmeticExpr(result)
-    )
-    or
-    result = v.(SemSsaExplicitUpdate).getSourceExpr() and
-    delta = D::fromFloat(0) and
-    not Lang::ignoreSsaReadAssignment(v)
-    or
-    result = Lang::specificSsaRead(v, delta)
-    or
-    result.(SemCopyValueExpr).getOperand() = semSsaRead(v, delta) and
-    not Lang::ignoreSsaReadCopy(result)
-    or
-    result.(SemStoreExpr).getOperand() = semSsaRead(v, delta)
-  }
-
-  /**
-   * Gets a condition that tests whether `v` equals `e + delta`.
-   *
-   * If the condition evaluates to `testIsTrue`:
-   * - `isEq = true`  : `v == e + delta`
-   * - `isEq = false` : `v != e + delta`
-   */
-  SemGuard semEqFlowCond(
-    SemSsaVariable v, SemExpr e, D::Delta delta, boolean isEq, boolean testIsTrue
-  ) {
-    exists(boolean eqpolarity |
-      result.isEquality(semSsaRead(v, delta), e, eqpolarity) and
-      (testIsTrue = true or testIsTrue = false) and
-      eqpolarity.booleanXor(testIsTrue).booleanNot() = isEq
-    )
-    or
-    exists(boolean testIsTrue0 |
-      semImplies_v2(result, testIsTrue, semEqFlowCond(v, e, delta, isEq, testIsTrue0), testIsTrue0)
-    )
-  }
-
-  /**
-   * Holds if `v` is an `SsaExplicitUpdate` that equals `e + delta`.
-   */
-  predicate semSsaUpdateStep(SemSsaExplicitUpdate v, SemExpr e, D::Delta delta) {
-    exists(SemExpr defExpr | defExpr = v.getSourceExpr() |
-      defExpr.(SemCopyValueExpr).getOperand() = e and delta = D::fromFloat(0)
-      or
-      defExpr.(SemStoreExpr).getOperand() = e and delta = D::fromFloat(0)
-      or
-      defExpr.(SemAddOneExpr).getOperand() = e and delta = D::fromFloat(1)
-      or
-      defExpr.(SemSubOneExpr).getOperand() = e and delta = D::fromFloat(-1)
-      or
-      e = defExpr and
-      not (
-        defExpr instanceof SemCopyValueExpr or
-        defExpr instanceof SemStoreExpr or
-        defExpr instanceof SemAddOneExpr or
-        defExpr instanceof SemSubOneExpr
-      ) and
-      delta = D::fromFloat(0)
-    )
-  }
-
-  /**
-   * Holds if `e1 + delta` equals `e2`.
-   */
-  predicate semValueFlowStep(SemExpr e2, SemExpr e1, D::Delta delta) {
-    e2.(SemCopyValueExpr).getOperand() = e1 and delta = D::fromFloat(0)
-    or
-    e2.(SemStoreExpr).getOperand() = e1 and delta = D::fromFloat(0)
-    or
-    e2.(SemAddOneExpr).getOperand() = e1 and delta = D::fromFloat(1)
-    or
-    e2.(SemSubOneExpr).getOperand() = e1 and delta = D::fromFloat(-1)
-    or
-    Lang::additionalValueFlowStep(e2, e1, delta)
-    or
-    exists(SemExpr x | e2.(SemAddExpr).hasOperands(e1, x) |
-      D::fromInt(x.(SemConstantIntegerExpr).getIntValue()) = delta
-    )
-    or
-    exists(SemExpr x, SemSubExpr sub |
-      e2 = sub and
-      sub.getLeftOperand() = e1 and
-      sub.getRightOperand() = x
-    |
-      D::fromInt(-x.(SemConstantIntegerExpr).getIntValue()) = delta
-    )
-  }
-
-  /**
-   * Gets the type used to track the specified expression's range information.
-   *
-   * Usually, this just `e.getSemType()`, but the language can override this to track immutable boxed
-   * primitive types as the underlying primitive type.
-   */
-  SemType getTrackedType(SemExpr e) {
-    result = Lang::getAlternateType(e)
-    or
-    not exists(Lang::getAlternateType(e)) and result = e.getSemType()
-  }
-
-  /**
-   * Gets the type used to track the specified source variable's range information.
-   *
-   * Usually, this just `e.getType()`, but the language can override this to track immutable boxed
-   * primitive types as the underlying primitive type.
-   */
-  SemType getTrackedTypeForSsaVariable(SemSsaVariable var) {
-    result = Lang::getAlternateTypeForSsaVariable(var)
-    or
-    not exists(Lang::getAlternateTypeForSsaVariable(var)) and result = var.getType()
-  }
+/**
+ * Gets an expression that equals `v - d`.
+ */
+SemExpr semSsaRead(SemSsaVariable v, int delta) {
+  // There are various language-specific extension points that can be removed once we no longer
+  // expect to match the original Java implementation's results exactly.
+  result = v.getAUse() and delta = 0
+  or
+  exists(int d1, SemConstantIntegerExpr c |
+    result.(SemAddExpr).hasOperands(semSsaRead(v, d1), c) and
+    delta = d1 - c.getIntValue() and
+    not Specific::ignoreSsaReadArithmeticExpr(result)
+  )
+  or
+  exists(SemSubExpr sub, int d1, SemConstantIntegerExpr c |
+    result = sub and
+    sub.getLeftOperand() = semSsaRead(v, d1) and
+    sub.getRightOperand() = c and
+    delta = d1 + c.getIntValue() and
+    not Specific::ignoreSsaReadArithmeticExpr(result)
+  )
+  or
+  result = v.(SemSsaExplicitUpdate).getSourceExpr() and
+  delta = 0 and
+  not Specific::ignoreSsaReadAssignment(v)
+  or
+  result = Specific::specificSsaRead(v, delta)
+  or
+  result.(SemCopyValueExpr).getOperand() = semSsaRead(v, delta) and
+  not Specific::ignoreSsaReadCopy(result)
+  or
+  result.(SemStoreExpr).getOperand() = semSsaRead(v, delta)
+}
+
+/**
+ * Gets a condition that tests whether `v` equals `e + delta`.
+ *
+ * If the condition evaluates to `testIsTrue`:
+ * - `isEq = true`  : `v == e + delta`
+ * - `isEq = false` : `v != e + delta`
+ */
+SemGuard semEqFlowCond(SemSsaVariable v, SemExpr e, int delta, boolean isEq, boolean testIsTrue) {
+  exists(boolean eqpolarity |
+    result.isEquality(semSsaRead(v, delta), e, eqpolarity) and
+    (testIsTrue = true or testIsTrue = false) and
+    eqpolarity.booleanXor(testIsTrue).booleanNot() = isEq
+  )
+  or
+  exists(boolean testIsTrue0 |
+    semImplies_v2(result, testIsTrue, semEqFlowCond(v, e, delta, isEq, testIsTrue0), testIsTrue0)
+  )
+}
+
+/**
+ * Holds if `v` is an `SsaExplicitUpdate` that equals `e + delta`.
+ */
+predicate semSsaUpdateStep(SemSsaExplicitUpdate v, SemExpr e, int delta) {
+  exists(SemExpr defExpr | defExpr = v.getSourceExpr() |
+    defExpr.(SemCopyValueExpr).getOperand() = e and delta = 0
+    or
+    defExpr.(SemStoreExpr).getOperand() = e and delta = 0
+    or
+    defExpr.(SemAddOneExpr).getOperand() = e and delta = 1
+    or
+    defExpr.(SemSubOneExpr).getOperand() = e and delta = -1
+    or
+    e = defExpr and
+    not (
+      defExpr instanceof SemCopyValueExpr or
+      defExpr instanceof SemStoreExpr or
+      defExpr instanceof SemAddOneExpr or
+      defExpr instanceof SemSubOneExpr
+    ) and
+    delta = 0
+  )
+}
+
+/**
+ * Holds if `e1 + delta` equals `e2`.
+ */
+predicate semValueFlowStep(SemExpr e2, SemExpr e1, int delta) {
+  e2.(SemCopyValueExpr).getOperand() = e1 and delta = 0
+  or
+  e2.(SemStoreExpr).getOperand() = e1 and delta = 0
+  or
+  e2.(SemAddOneExpr).getOperand() = e1 and delta = 1
+  or
+  e2.(SemSubOneExpr).getOperand() = e1 and delta = -1
+  or
+  Specific::additionalValueFlowStep(e2, e1, delta)
+  or
+  exists(SemExpr x | e2.(SemAddExpr).hasOperands(e1, x) |
+    x.(SemConstantIntegerExpr).getIntValue() = delta
+  )
+  or
+  exists(SemExpr x, SemSubExpr sub |
+    e2 = sub and
+    sub.getLeftOperand() = e1 and
+    sub.getRightOperand() = x
+  |
+    x.(SemConstantIntegerExpr).getIntValue() = -delta
+  )
+}
+
+/**
+ * Gets the type used to track the specified expression's range information.
+ *
+ * Usually, this just `e.getSemType()`, but the language can override this to track immutable boxed
+ * primitive types as the underlying primitive type.
+ */
+SemType getTrackedType(SemExpr e) {
+  result = Specific::getAlternateType(e)
+  or
+  not exists(Specific::getAlternateType(e)) and result = e.getSemType()
+}
+
+/**
+ * Gets the type used to track the specified source variable's range information.
+ *
+ * Usually, this just `e.getType()`, but the language can override this to track immutable boxed
+ * primitive types as the underlying primitive type.
+ */
+SemType getTrackedTypeForSsaVariable(SemSsaVariable var) {
+  result = Specific::getAlternateTypeForSsaVariable(var)
+  or
+  not exists(Specific::getAlternateTypeForSsaVariable(var)) and result = var.getType()
 }

cpp/ql/lib/experimental/semmle/code/cpp/semantic/analysis/SignAnalysisCommon.qll

@@ -6,494 +6,496 @@
  * three-valued domain `{negative, zero, positive}`.
  */
 
-private import RangeAnalysisStage
 private import SignAnalysisSpecific as Specific
 private import experimental.semmle.code.cpp.semantic.Semantic
 private import ConstantAnalysis
 private import RangeUtils
 private import Sign
 
-module SignAnalysis<DeltaSig D, UtilSig<D> Utils> {
-  /**
-   * An SSA definition for which the analysis can compute the sign.
-   *
-   * The actual computation of the sign is done in an override of the `getSign()` predicate. The
-   * charpred of any subclass must _not_ invoke `getSign()`, directly or indirectly. This ensures
-   * that the charpred does not introduce negative recursion. The `getSign()` predicate may be
-   * recursive.
-   */
-  abstract private class SignDef instanceof SemSsaVariable {
-    final string toString() { result = super.toString() }
+/**
+ * An SSA definition for which the analysis can compute the sign.
+ *
+ * The actual computation of the sign is done in an override of the `getSign()` predicate. The
+ * charpred of any subclass must _not_ invoke `getSign()`, directly or indirectly. This ensures
+ * that the charpred does not introduce negative recursion. The `getSign()` predicate may be
+ * recursive.
+ */
+abstract private class SignDef instanceof SemSsaVariable {
+  final string toString() { result = super.toString() }
 
-    /** Gets the possible signs of this SSA definition. */
-    abstract Sign getSign();
-  }
+  /** Gets the possible signs of this SSA definition. */
+  abstract Sign getSign();
+}
 
-  /** An SSA definition whose sign is computed based on standard flow. */
-  abstract private class FlowSignDef extends SignDef {
-    abstract override Sign getSign();
-  }
+/** An SSA definition whose sign is computed based on standard flow. */
+abstract private class FlowSignDef extends SignDef {
+  abstract override Sign getSign();
+}
 
-  /** An SSA definition whose sign is determined by the sign of that definitions source expression. */
-  private class ExplicitSignDef extends FlowSignDef instanceof SemSsaExplicitUpdate {
-    final override Sign getSign() { result = semExprSign(super.getSourceExpr()) }
-  }
+/** An SSA definition whose sign is determined by the sign of that definitions source expression. */
+private class ExplicitSignDef extends FlowSignDef {
+  SemSsaExplicitUpdate update;
 
-  /** An SSA Phi definition, whose sign is the union of the signs of its inputs. */
-  private class PhiSignDef extends FlowSignDef instanceof SemSsaPhiNode {
-    final override Sign getSign() {
-      exists(SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge |
-        edge.phiInput(this, inp) and
-        result = semSsaSign(inp, edge)
-      )
-    }
-  }
+  ExplicitSignDef() { update = this }
 
-  /** An SSA definition whose sign is computed by a language-specific implementation. */
-  abstract class CustomSignDef extends SignDef {
-    abstract override Sign getSign();
-  }
+  final override Sign getSign() { result = semExprSign(update.getSourceExpr()) }
+}
 
-  /**
-   * An expression for which the analysis can compute the sign.
-   *
-   * The actual computation of the sign is done in an override of the `getSign()` predicate. The
-   * charpred of any subclass must _not_ invoke `getSign()`, directly or indirectly. This ensures
-   * that the charpred does not introduce negative recursion. The `getSign()` predicate may be
-   * recursive.
-   *
-   * Concrete implementations extend one of the following subclasses:
-   * - `ConstantSignExpr`, for expressions with a compile-time constant value.
-   * - `FlowSignExpr`, for expressions whose sign can be computed from the signs of their operands.
-   * - `CustomsignExpr`, for expressions whose sign can be computed by a language-specific
-   *   implementation.
-   *
-   * If the same expression matches more than one of the above subclasses, the sign is computed as
-   * follows:
-   * - The sign of a `ConstantSignExpr` is computed solely from `ConstantSignExpr.getSign()`,
-   *   regardless of any other subclasses.
-   * - If a non-`ConstantSignExpr` expression matches exactly one of `FlowSignExpr` or
-   *   `CustomSignExpr`, the sign is computed by that class' `getSign()` predicate.
-   * - If a non-`ConstantSignExpr` expression matches both `FlowSignExpr` and `CustomSignExpr`, the
-   *   sign is the _intersection_ of the signs of those two classes' `getSign()` predicates. Thus,
-   *   both classes have the opportunity to _restrict_ the set of possible signs, not to generate new
-   *   possible signs.
-   * - If an expression does not match any of the three subclasses, then it can have any sign.
-   *
-   * Note that the `getSign()` predicate is introduced only in subclasses of `SignExpr`.
-   */
-  abstract class SignExpr instanceof SemExpr {
-    SignExpr() { not Specific::ignoreExprSign(this) }
+/** An SSA Phi definition, whose sign is the union of the signs of its inputs. */
+private class PhiSignDef extends FlowSignDef {
+  SemSsaPhiNode phi;
 
-    final string toString() { result = super.toString() }
+  PhiSignDef() { phi = this }
 
-    abstract Sign getSign();
-  }
-
-  /** An expression whose sign is determined by its constant numeric value. */
-  private class ConstantSignExpr extends SignExpr {
-    ConstantSignExpr() {
-      this instanceof SemConstantIntegerExpr or
-      exists(this.(SemNumericLiteralExpr).getApproximateFloatValue())
-    }
-
-    final override Sign getSign() {
-      exists(int i | this.(SemConstantIntegerExpr).getIntValue() = i |
-        i < 0 and result = TNeg()
-        or
-        i = 0 and result = TZero()
-        or
-        i > 0 and result = TPos()
-      )
-      or
-      not exists(this.(SemConstantIntegerExpr).getIntValue()) and
-      exists(float f | f = this.(SemNumericLiteralExpr).getApproximateFloatValue() |
-        f < 0 and result = TNeg()
-        or
-        f = 0 and result = TZero()
-        or
-        f > 0 and result = TPos()
-      )
-    }
-  }
-
-  abstract private class NonConstantSignExpr extends SignExpr {
-    NonConstantSignExpr() { not this instanceof ConstantSignExpr }
-
-    final override Sign getSign() {
-      // The result is the _intersection_ of the signs computed from flow and by the language.
-      (result = this.(FlowSignExpr).getSignRestriction() or not this instanceof FlowSignExpr) and
-      (result = this.(CustomSignExpr).getSignRestriction() or not this instanceof CustomSignExpr)
-    }
-  }
-
-  /** An expression whose sign is computed from the signs of its operands. */
-  abstract private class FlowSignExpr extends NonConstantSignExpr {
-    abstract Sign getSignRestriction();
-  }
-
-  /** An expression whose sign is computed by a language-specific implementation. */
-  abstract class CustomSignExpr extends NonConstantSignExpr {
-    abstract Sign getSignRestriction();
-  }
-
-  /** An expression whose sign is unknown. */
-  private class UnknownSignExpr extends SignExpr {
-    UnknownSignExpr() {
-      not this instanceof FlowSignExpr and
-      not this instanceof CustomSignExpr and
-      not this instanceof ConstantSignExpr and
-      (
-        // Only track numeric types.
-        Utils::getTrackedType(this) instanceof SemNumericType
-        or
-        // Unless the language says to track this expression anyway.
-        Specific::trackUnknownNonNumericExpr(this)
-      )
-    }
-
-    final override Sign getSign() { semAnySign(result) }
-  }
-
-  /**
-   * A `Load` expression whose sign is computed from the sign of its SSA definition, restricted by
-   * inference from any intervening guards.
-   */
-  class UseSignExpr extends FlowSignExpr {
-    SemSsaVariable v;
-
-    UseSignExpr() { v.getAUse() = this }
-
-    override Sign getSignRestriction() {
-      // Propagate via SSA
-      // Propagate the sign from the def of `v`, incorporating any inference from guards.
-      result = semSsaSign(v, any(SemSsaReadPositionBlock bb | bb.getAnExpr() = this))
-      or
-      // No block for this read. Just use the sign of the def.
-      // REVIEW: How can this happen?
-      not exists(SemSsaReadPositionBlock bb | bb.getAnExpr() = this) and
-      result = semSsaDefSign(v)
-    }
-  }
-
-  /** A binary expression whose sign is computed from the signs of its operands. */
-  private class BinarySignExpr extends FlowSignExpr {
-    SemBinaryExpr binary;
-
-    BinarySignExpr() { binary = this }
-
-    override Sign getSignRestriction() {
-      exists(SemExpr left, SemExpr right |
-        binaryExprOperands(binary, left, right) and
-        result =
-          semExprSign(pragma[only_bind_out](left))
-              .applyBinaryOp(semExprSign(pragma[only_bind_out](right)), binary.getOpcode())
-      )
-      or
-      exists(SemDivExpr div | div = binary |
-        result = semExprSign(div.getLeftOperand()) and
-        result != TZero() and
-        div.getRightOperand().(SemFloatingPointLiteralExpr).getFloatValue() = 0
-      )
-    }
-  }
-
-  pragma[nomagic]
-  private predicate binaryExprOperands(SemBinaryExpr binary, SemExpr left, SemExpr right) {
-    binary.getLeftOperand() = left and binary.getRightOperand() = right
-  }
-
-  /**
-   * A `Convert`, `Box`, or `Unbox` expression.
-   */
-  private class SemCastExpr instanceof SemUnaryExpr {
-    string toString() { result = super.toString() }
-
-    SemCastExpr() {
-      this instanceof SemConvertExpr
-      or
-      this instanceof SemBoxExpr
-      or
-      this instanceof SemUnboxExpr
-    }
-  }
-
-  /** A unary expression whose sign is computed from the sign of its operand. */
-  private class UnarySignExpr extends FlowSignExpr {
-    SemUnaryExpr unary;
-
-    UnarySignExpr() { unary = this and not this instanceof SemCastExpr }
-
-    override Sign getSignRestriction() {
-      result =
-        semExprSign(pragma[only_bind_out](unary.getOperand())).applyUnaryOp(unary.getOpcode())
-    }
-  }
-
-  /**
-   * A `Convert`, `Box`, or `Unbox` expression, whose sign is computed based on
-   * the sign of its operand and the source and destination types.
-   */
-  abstract private class CastSignExpr extends FlowSignExpr {
-    SemUnaryExpr cast;
-
-    CastSignExpr() { cast = this and cast instanceof SemCastExpr }
-
-    override Sign getSignRestriction() { result = semExprSign(cast.getOperand()) }
-  }
-
-  /**
-   * A `Convert` expression.
-   */
-  private class ConvertSignExpr extends CastSignExpr {
-    override SemConvertExpr cast;
-  }
-
-  /**
-   * A `Box` expression.
-   */
-  private class BoxSignExpr extends CastSignExpr {
-    override SemBoxExpr cast;
-  }
-
-  /**
-   * An `Unbox` expression.
-   */
-  private class UnboxSignExpr extends CastSignExpr {
-    override SemUnboxExpr cast;
-
-    UnboxSignExpr() {
-      exists(SemType fromType | fromType = Utils::getTrackedType(cast.getOperand()) |
-        // Only numeric source types are handled here.
-        fromType instanceof SemNumericType
-      )
-    }
-  }
-
-  private predicate unknownSign(SemExpr e) { e instanceof UnknownSignExpr }
-
-  /**
-   * Holds if `lowerbound` is a lower bound for `v` at `pos`. This is restricted
-   * to only include bounds for which we might determine a sign.
-   */
-  private predicate lowerBound(
-    SemExpr lowerbound, SemSsaVariable v, SemSsaReadPosition pos, boolean isStrict
-  ) {
-    exists(boolean testIsTrue, SemRelationalExpr comp |
-      pos.hasReadOfVar(v) and
-      semGuardControlsSsaRead(semGetComparisonGuard(comp), pos, testIsTrue) and
-      not unknownSign(lowerbound)
-    |
-      testIsTrue = true and
-      comp.getLesserOperand() = lowerbound and
-      comp.getGreaterOperand() = Utils::semSsaRead(v, D::fromInt(0)) and
-      (if comp.isStrict() then isStrict = true else isStrict = false)
-      or
-      testIsTrue = false and
-      comp.getGreaterOperand() = lowerbound and
-      comp.getLesserOperand() = Utils::semSsaRead(v, D::fromInt(0)) and
-      (if comp.isStrict() then isStrict = false else isStrict = true)
+  final override Sign getSign() {
+    exists(SemSsaVariable inp, SemSsaReadPositionPhiInputEdge edge |
+      edge.phiInput(phi, inp) and
+      result = semSsaSign(inp, edge)
     )
   }
-
-  /**
-   * Holds if `upperbound` is an upper bound for `v` at `pos`. This is restricted
-   * to only include bounds for which we might determine a sign.
-   */
-  private predicate upperBound(
-    SemExpr upperbound, SemSsaVariable v, SemSsaReadPosition pos, boolean isStrict
-  ) {
-    exists(boolean testIsTrue, SemRelationalExpr comp |
-      pos.hasReadOfVar(v) and
-      semGuardControlsSsaRead(semGetComparisonGuard(comp), pos, testIsTrue) and
-      not unknownSign(upperbound)
-    |
-      testIsTrue = true and
-      comp.getGreaterOperand() = upperbound and
-      comp.getLesserOperand() = Utils::semSsaRead(v, D::fromInt(0)) and
-      (if comp.isStrict() then isStrict = true else isStrict = false)
-      or
-      testIsTrue = false and
-      comp.getLesserOperand() = upperbound and
-      comp.getGreaterOperand() = Utils::semSsaRead(v, D::fromInt(0)) and
-      (if comp.isStrict() then isStrict = false else isStrict = true)
-    )
-  }
-
-  /**
-   * Holds if `eqbound` is an equality/inequality for `v` at `pos`. This is
-   * restricted to only include bounds for which we might determine a sign. The
-   * boolean `isEq` gives the polarity:
-   *  - `isEq = true` : `v = eqbound`
-   *  - `isEq = false` : `v != eqbound`
-   */
-  private predicate eqBound(SemExpr eqbound, SemSsaVariable v, SemSsaReadPosition pos, boolean isEq) {
-    exists(SemGuard guard, boolean testIsTrue, boolean polarity |
-      pos.hasReadOfVar(v) and
-      semGuardControlsSsaRead(guard, pos, testIsTrue) and
-      guard.isEquality(eqbound, Utils::semSsaRead(v, D::fromInt(0)), polarity) and
-      isEq = polarity.booleanXor(testIsTrue).booleanNot() and
-      not unknownSign(eqbound)
-    )
-  }
-
-  /**
-   * Holds if `bound` is a bound for `v` at `pos` that needs to be positive in
-   * order for `v` to be positive.
-   */
-  private predicate posBound(SemExpr bound, SemSsaVariable v, SemSsaReadPosition pos) {
-    upperBound(bound, v, pos, _) or
-    eqBound(bound, v, pos, true)
-  }
-
-  /**
-   * Holds if `bound` is a bound for `v` at `pos` that needs to be negative in
-   * order for `v` to be negative.
-   */
-  private predicate negBound(SemExpr bound, SemSsaVariable v, SemSsaReadPosition pos) {
-    lowerBound(bound, v, pos, _) or
-    eqBound(bound, v, pos, true)
-  }
-
-  /**
-   * Holds if `bound` is a bound for `v` at `pos` that can restrict whether `v`
-   * can be zero.
-   */
-  private predicate zeroBound(SemExpr bound, SemSsaVariable v, SemSsaReadPosition pos) {
-    lowerBound(bound, v, pos, _) or
-    upperBound(bound, v, pos, _) or
-    eqBound(bound, v, pos, _)
-  }
-
-  /** Holds if `bound` allows `v` to be positive at `pos`. */
-  private predicate posBoundOk(SemExpr bound, SemSsaVariable v, SemSsaReadPosition pos) {
-    posBound(bound, v, pos) and TPos() = semExprSign(bound)
-  }
-
-  /** Holds if `bound` allows `v` to be negative at `pos`. */
-  private predicate negBoundOk(SemExpr bound, SemSsaVariable v, SemSsaReadPosition pos) {
-    negBound(bound, v, pos) and TNeg() = semExprSign(bound)
-  }
-
-  /** Holds if `bound` allows `v` to be zero at `pos`. */
-  private predicate zeroBoundOk(SemExpr bound, SemSsaVariable v, SemSsaReadPosition pos) {
-    lowerBound(bound, v, pos, _) and TNeg() = semExprSign(bound)
-    or
-    lowerBound(bound, v, pos, false) and TZero() = semExprSign(bound)
-    or
-    upperBound(bound, v, pos, _) and TPos() = semExprSign(bound)
-    or
-    upperBound(bound, v, pos, false) and TZero() = semExprSign(bound)
-    or
-    eqBound(bound, v, pos, true) and TZero() = semExprSign(bound)
-    or
-    eqBound(bound, v, pos, false) and TZero() != semExprSign(bound)
-  }
-
-  /**
-   * Holds if there is a bound that might restrict whether `v` has the sign `s`
-   * at `pos`.
-   */
-  private predicate hasGuard(SemSsaVariable v, SemSsaReadPosition pos, Sign s) {
-    s = TPos() and posBound(_, v, pos)
-    or
-    s = TNeg() and negBound(_, v, pos)
-    or
-    s = TZero() and zeroBound(_, v, pos)
-  }
-
-  /**
-   * Gets a possible sign of `v` at `pos` based on its definition, where the sign
-   * might be ruled out by a guard.
-   */
-  pragma[noinline]
-  private Sign guardedSsaSign(SemSsaVariable v, SemSsaReadPosition pos) {
-    result = semSsaDefSign(v) and
-    pos.hasReadOfVar(v) and
-    hasGuard(v, pos, result)
-  }
-
-  /**
-   * Gets a possible sign of `v` at `pos` based on its definition, where no guard
-   * can rule it out.
-   */
-  pragma[noinline]
-  private Sign unguardedSsaSign(SemSsaVariable v, SemSsaReadPosition pos) {
-    result = semSsaDefSign(v) and
-    pos.hasReadOfVar(v) and
-    not hasGuard(v, pos, result)
-  }
-
-  /**
-   * Gets a possible sign of `v` at read position `pos`, where a guard could have
-   * ruled out the sign but does not.
-   * This does not check that the definition of `v` also allows the sign.
-   */
-  private Sign guardedSsaSignOk(SemSsaVariable v, SemSsaReadPosition pos) {
-    result = TPos() and
-    forex(SemExpr bound | posBound(bound, v, pos) | posBoundOk(bound, v, pos))
-    or
-    result = TNeg() and
-    forex(SemExpr bound | negBound(bound, v, pos) | negBoundOk(bound, v, pos))
-    or
-    result = TZero() and
-    forex(SemExpr bound | zeroBound(bound, v, pos) | zeroBoundOk(bound, v, pos))
-  }
-
-  /** Gets a possible sign for `v` at `pos`. */
-  private Sign semSsaSign(SemSsaVariable v, SemSsaReadPosition pos) {
-    result = unguardedSsaSign(v, pos)
-    or
-    result = guardedSsaSign(v, pos) and
-    result = guardedSsaSignOk(v, pos)
-  }
-
-  /** Gets a possible sign for `v`. */
-  pragma[nomagic]
-  Sign semSsaDefSign(SemSsaVariable v) { result = v.(SignDef).getSign() }
-
-  /** Gets a possible sign for `e`. */
-  cached
-  Sign semExprSign(SemExpr e) {
-    exists(Sign s | s = e.(SignExpr).getSign() |
-      if
-        Utils::getTrackedType(e) instanceof SemUnsignedIntegerType and
-        s = TNeg() and
-        not Specific::ignoreTypeRestrictions(e)
-      then result = TPos()
-      else result = s
-    )
-  }
-
-  /**
-   * Dummy predicate that holds for any sign. This is added to improve readability
-   * of cases where the sign is unrestricted.
-   */
-  predicate semAnySign(Sign s) { any() }
-
-  /** Holds if `e` can be positive and cannot be negative. */
-  predicate semPositive(SemExpr e) {
-    semExprSign(e) = TPos() and
-    not semExprSign(e) = TNeg()
-  }
-
-  /** Holds if `e` can be negative and cannot be positive. */
-  predicate semNegative(SemExpr e) {
-    semExprSign(e) = TNeg() and
-    not semExprSign(e) = TPos()
-  }
-
-  /** Holds if `e` is strictly positive. */
-  predicate semStrictlyPositive(SemExpr e) {
-    semExprSign(e) = TPos() and
-    not semExprSign(e) = TNeg() and
-    not semExprSign(e) = TZero()
-  }
-
-  /** Holds if `e` is strictly negative. */
-  predicate semStrictlyNegative(SemExpr e) {
-    semExprSign(e) = TNeg() and
-    not semExprSign(e) = TPos() and
-    not semExprSign(e) = TZero()
-  }
+}
+
+/** An SSA definition whose sign is computed by a language-specific implementation. */
+abstract class CustomSignDef extends SignDef {
+  abstract override Sign getSign();
+}
+
+/**
+ * An expression for which the analysis can compute the sign.
+ *
+ * The actual computation of the sign is done in an override of the `getSign()` predicate. The
+ * charpred of any subclass must _not_ invoke `getSign()`, directly or indirectly. This ensures
+ * that the charpred does not introduce negative recursion. The `getSign()` predicate may be
+ * recursive.
+ *
+ * Concrete implementations extend one of the following subclasses:
+ * - `ConstantSignExpr`, for expressions with a compile-time constant value.
+ * - `FlowSignExpr`, for expressions whose sign can be computed from the signs of their operands.
+ * - `CustomsignExpr`, for expressions shose sign can be computed by a language-specific
+ *   implementation.
+ *
+ * If the same expression matches more than one of the above subclasses, the sign is computed as
+ * follows:
+ * - The sign of a `ConstantSignExpr` is computed solely from `ConstantSignExpr.getSign()`,
+ *   regardless of any other subclasses.
+ * - If a non-`ConstantSignExpr` expression matches exactly one of `FlowSignExpr` or
+ *   `CustomSignExpr`, the sign is computed by that class' `getSign()` predicate.
+ * - If a non-`ConstantSignExpr` expression matches both `FlowSignExpr` and `CustomSignExpr`, the
+ *   sign is the _intersection_ of the signs of those two classes' `getSign()` predicates. Thus,
+ *   both classes have the opportunity to _restrict_ the set of possible signs, not to generate new
+ *   possible signs.
+ * - If an expression does not match any of the three subclasses, then it can have any sign.
+ *
+ * Note that the `getSign()` predicate is introduced only in subclasses of `SignExpr`.
+ */
+abstract class SignExpr instanceof SemExpr {
+  SignExpr() { not Specific::ignoreExprSign(this) }
+
+  final string toString() { result = super.toString() }
+
+  abstract Sign getSign();
+}
+
+/** An expression whose sign is determined by its constant numeric value. */
+private class ConstantSignExpr extends SignExpr {
+  ConstantSignExpr() {
+    this instanceof SemConstantIntegerExpr or
+    exists(this.(SemNumericLiteralExpr).getApproximateFloatValue())
+  }
+
+  final override Sign getSign() {
+    exists(int i | this.(SemConstantIntegerExpr).getIntValue() = i |
+      i < 0 and result = TNeg()
+      or
+      i = 0 and result = TZero()
+      or
+      i > 0 and result = TPos()
+    )
+    or
+    not exists(this.(SemConstantIntegerExpr).getIntValue()) and
+    exists(float f | f = this.(SemNumericLiteralExpr).getApproximateFloatValue() |
+      f < 0 and result = TNeg()
+      or
+      f = 0 and result = TZero()
+      or
+      f > 0 and result = TPos()
+    )
+  }
+}
+
+abstract private class NonConstantSignExpr extends SignExpr {
+  NonConstantSignExpr() { not this instanceof ConstantSignExpr }
+
+  final override Sign getSign() {
+    // The result is the _intersection_ of the signs computed from flow and by the language.
+    (result = this.(FlowSignExpr).getSignRestriction() or not this instanceof FlowSignExpr) and
+    (result = this.(CustomSignExpr).getSignRestriction() or not this instanceof CustomSignExpr)
+  }
+}
+
+/** An expression whose sign is computed from the signs of its operands. */
+abstract private class FlowSignExpr extends NonConstantSignExpr {
+  abstract Sign getSignRestriction();
+}
+
+/** An expression whose sign is computed by a language-specific implementation. */
+abstract class CustomSignExpr extends NonConstantSignExpr {
+  abstract Sign getSignRestriction();
+}
+
+/** An expression whose sign is unknown. */
+private class UnknownSignExpr extends SignExpr {
+  UnknownSignExpr() {
+    not this instanceof FlowSignExpr and
+    not this instanceof CustomSignExpr and
+    not this instanceof ConstantSignExpr and
+    (
+      // Only track numeric types.
+      getTrackedType(this) instanceof SemNumericType
+      or
+      // Unless the language says to track this expression anyway.
+      Specific::trackUnknownNonNumericExpr(this)
+    )
+  }
+
+  final override Sign getSign() { semAnySign(result) }
+}
+
+/**
+ * A `Load` expression whose sign is computed from the sign of its SSA definition, restricted by
+ * inference from any intervening guards.
+ */
+class UseSignExpr extends FlowSignExpr {
+  SemSsaVariable v;
+
+  UseSignExpr() { v.getAUse() = this }
+
+  override Sign getSignRestriction() {
+    // Propagate via SSA
+    // Propagate the sign from the def of `v`, incorporating any inference from guards.
+    result = semSsaSign(v, any(SemSsaReadPositionBlock bb | bb.getAnExpr() = this))
+    or
+    // No block for this read. Just use the sign of the def.
+    // REVIEW: How can this happen?
+    not exists(SemSsaReadPositionBlock bb | bb.getAnExpr() = this) and
+    result = semSsaDefSign(v)
+  }
+}
+
+/** A binary expression whose sign is computed from the signs of its operands. */
+private class BinarySignExpr extends FlowSignExpr {
+  SemBinaryExpr binary;
+
+  BinarySignExpr() { binary = this }
+
+  override Sign getSignRestriction() {
+    exists(SemExpr left, SemExpr right |
+      binaryExprOperands(binary, left, right) and
+      result =
+        semExprSign(pragma[only_bind_out](left))
+            .applyBinaryOp(semExprSign(pragma[only_bind_out](right)), binary.getOpcode())
+    )
+    or
+    exists(SemDivExpr div | div = binary |
+      result = semExprSign(div.getLeftOperand()) and
+      result != TZero() and
+      div.getRightOperand().(SemFloatingPointLiteralExpr).getFloatValue() = 0
+    )
+  }
+}
+
+pragma[nomagic]
+private predicate binaryExprOperands(SemBinaryExpr binary, SemExpr left, SemExpr right) {
+  binary.getLeftOperand() = left and binary.getRightOperand() = right
+}
+
+/**
+ * A `Convert`, `Box`, or `Unbox` expression.
+ */
+private class SemCastExpr extends SemUnaryExpr {
+  SemCastExpr() {
+    this instanceof SemConvertExpr
+    or
+    this instanceof SemBoxExpr
+    or
+    this instanceof SemUnboxExpr
+  }
+}
+
+/** A unary expression whose sign is computed from the sign of its operand. */
+private class UnarySignExpr extends FlowSignExpr {
+  SemUnaryExpr unary;
+
+  UnarySignExpr() { unary = this and not this instanceof SemCastExpr }
+
+  override Sign getSignRestriction() {
+    result = semExprSign(pragma[only_bind_out](unary.getOperand())).applyUnaryOp(unary.getOpcode())
+  }
+}
+
+/**
+ * A `Convert`, `Box`, or `Unbox` expression, whose sign is computed based on
+ * the sign of its operand and the source and destination types.
+ */
+abstract private class CastSignExpr extends FlowSignExpr {
+  SemUnaryExpr cast;
+
+  CastSignExpr() { cast = this and cast instanceof SemCastExpr }
+
+  override Sign getSignRestriction() { result = semExprSign(cast.getOperand()) }
+}
+
+/**
+ * A `Convert` expression.
+ */
+private class ConvertSignExpr extends CastSignExpr {
+  override SemConvertExpr cast;
+}
+
+/**
+ * A `Box` expression.
+ */
+private class BoxSignExpr extends CastSignExpr {
+  override SemBoxExpr cast;
+}
+
+/**
+ * An `Unbox` expression.
+ */
+private class UnboxSignExpr extends CastSignExpr {
+  override SemUnboxExpr cast;
+
+  UnboxSignExpr() {
+    exists(SemType fromType | fromType = getTrackedType(cast.getOperand()) |
+      // Only numeric source types are handled here.
+      fromType instanceof SemNumericType
+    )
+  }
+}
+
+private predicate unknownSign(SemExpr e) { e instanceof UnknownSignExpr }
+
+/**
+ * Holds if `lowerbound` is a lower bound for `v` at `pos`. This is restricted
+ * to only include bounds for which we might determine a sign.
+ */
+private predicate lowerBound(
+  SemExpr lowerbound, SemSsaVariable v, SemSsaReadPosition pos, boolean isStrict
+) {
+  exists(boolean testIsTrue, SemRelationalExpr comp |
+    pos.hasReadOfVar(v) and
+    semGuardControlsSsaRead(semGetComparisonGuard(comp), pos, testIsTrue) and
+    not unknownSign(lowerbound)
+  |
+    testIsTrue = true and
+    comp.getLesserOperand() = lowerbound and
+    comp.getGreaterOperand() = semSsaRead(v, 0) and
+    (if comp.isStrict() then isStrict = true else isStrict = false)
+    or
+    testIsTrue = false and
+    comp.getGreaterOperand() = lowerbound and
+    comp.getLesserOperand() = semSsaRead(v, 0) and
+    (if comp.isStrict() then isStrict = false else isStrict = true)
+  )
+}
+
+/**
+ * Holds if `upperbound` is an upper bound for `v` at `pos`. This is restricted
+ * to only include bounds for which we might determine a sign.
+ */
+private predicate upperBound(
+  SemExpr upperbound, SemSsaVariable v, SemSsaReadPosition pos, boolean isStrict
+) {
+  exists(boolean testIsTrue, SemRelationalExpr comp |
+    pos.hasReadOfVar(v) and
+    semGuardControlsSsaRead(semGetComparisonGuard(comp), pos, testIsTrue) and
+    not unknownSign(upperbound)
+  |
+    testIsTrue = true and
+    comp.getGreaterOperand() = upperbound and
+    comp.getLesserOperand() = semSsaRead(v, 0) and
+    (if comp.isStrict() then isStrict = true else isStrict = false)
+    or
+    testIsTrue = false and
+    comp.getLesserOperand() = upperbound and
+    comp.getGreaterOperand() = semSsaRead(v, 0) and
+    (if comp.isStrict() then isStrict = false else isStrict = true)
+  )
+}
+
+/**
+ * Holds if `eqbound` is an equality/inequality for `v` at `pos`. This is
+ * restricted to only include bounds for which we might determine a sign. The
+ * boolean `isEq` gives the polarity:
+ *  - `isEq = true` : `v = eqbound`
+ *  - `isEq = false` : `v != eqbound`
+ */
+private predicate eqBound(SemExpr eqbound, SemSsaVariable v, SemSsaReadPosition pos, boolean isEq) {
+  exists(SemGuard guard, boolean testIsTrue, boolean polarity |
+    pos.hasReadOfVar(v) and
+    semGuardControlsSsaRead(guard, pos, testIsTrue) and
+    guard.isEquality(eqbound, semSsaRead(v, 0), polarity) and
+    isEq = polarity.booleanXor(testIsTrue).booleanNot() and
+    not unknownSign(eqbound)
+  )
+}
+
+/**
+ * Holds if `bound` is a bound for `v` at `pos` that needs to be positive in
+ * order for `v` to be positive.
+ */
+private predicate posBound(SemExpr bound, SemSsaVariable v, SemSsaReadPosition pos) {
+  upperBound(bound, v, pos, _) or
+  eqBound(bound, v, pos, true)
+}
+
+/**
+ * Holds if `bound` is a bound for `v` at `pos` that needs to be negative in
+ * order for `v` to be negative.
+ */
+private predicate negBound(SemExpr bound, SemSsaVariable v, SemSsaReadPosition pos) {
+  lowerBound(bound, v, pos, _) or
+  eqBound(bound, v, pos, true)
+}
+
+/**
+ * Holds if `bound` is a bound for `v` at `pos` that can restrict whether `v`
+ * can be zero.
+ */
+private predicate zeroBound(SemExpr bound, SemSsaVariable v, SemSsaReadPosition pos) {
+  lowerBound(bound, v, pos, _) or
+  upperBound(bound, v, pos, _) or
+  eqBound(bound, v, pos, _)
+}
+
+/** Holds if `bound` allows `v` to be positive at `pos`. */
+private predicate posBoundOk(SemExpr bound, SemSsaVariable v, SemSsaReadPosition pos) {
+  posBound(bound, v, pos) and TPos() = semExprSign(bound)
+}
+
+/** Holds if `bound` allows `v` to be negative at `pos`. */
+private predicate negBoundOk(SemExpr bound, SemSsaVariable v, SemSsaReadPosition pos) {
+  negBound(bound, v, pos) and TNeg() = semExprSign(bound)
+}
+
+/** Holds if `bound` allows `v` to be zero at `pos`. */
+private predicate zeroBoundOk(SemExpr bound, SemSsaVariable v, SemSsaReadPosition pos) {
+  lowerBound(bound, v, pos, _) and TNeg() = semExprSign(bound)
+  or
+  lowerBound(bound, v, pos, false) and TZero() = semExprSign(bound)
+  or
+  upperBound(bound, v, pos, _) and TPos() = semExprSign(bound)
+  or
+  upperBound(bound, v, pos, false) and TZero() = semExprSign(bound)
+  or
+  eqBound(bound, v, pos, true) and TZero() = semExprSign(bound)
+  or
+  eqBound(bound, v, pos, false) and TZero() != semExprSign(bound)
+}
+
+/**
+ * Holds if there is a bound that might restrict whether `v` has the sign `s`
+ * at `pos`.
+ */
+private predicate hasGuard(SemSsaVariable v, SemSsaReadPosition pos, Sign s) {
+  s = TPos() and posBound(_, v, pos)
+  or
+  s = TNeg() and negBound(_, v, pos)
+  or
+  s = TZero() and zeroBound(_, v, pos)
+}
+
+/**
+ * Gets a possible sign of `v` at `pos` based on its definition, where the sign
+ * might be ruled out by a guard.
+ */
+pragma[noinline]
+private Sign guardedSsaSign(SemSsaVariable v, SemSsaReadPosition pos) {
+  result = semSsaDefSign(v) and
+  pos.hasReadOfVar(v) and
+  hasGuard(v, pos, result)
+}
+
+/**
+ * Gets a possible sign of `v` at `pos` based on its definition, where no guard
+ * can rule it out.
+ */
+pragma[noinline]
+private Sign unguardedSsaSign(SemSsaVariable v, SemSsaReadPosition pos) {
+  result = semSsaDefSign(v) and
+  pos.hasReadOfVar(v) and
+  not hasGuard(v, pos, result)
+}
+
+/**
+ * Gets a possible sign of `v` at read position `pos`, where a guard could have
+ * ruled out the sign but does not.
+ * This does not check that the definition of `v` also allows the sign.
+ */
+private Sign guardedSsaSignOk(SemSsaVariable v, SemSsaReadPosition pos) {
+  result = TPos() and
+  forex(SemExpr bound | posBound(bound, v, pos) | posBoundOk(bound, v, pos))
+  or
+  result = TNeg() and
+  forex(SemExpr bound | negBound(bound, v, pos) | negBoundOk(bound, v, pos))
+  or
+  result = TZero() and
+  forex(SemExpr bound | zeroBound(bound, v, pos) | zeroBoundOk(bound, v, pos))
+}
+
+/** Gets a possible sign for `v` at `pos`. */
+private Sign semSsaSign(SemSsaVariable v, SemSsaReadPosition pos) {
+  result = unguardedSsaSign(v, pos)
+  or
+  result = guardedSsaSign(v, pos) and
+  result = guardedSsaSignOk(v, pos)
+}
+
+/** Gets a possible sign for `v`. */
+pragma[nomagic]
+Sign semSsaDefSign(SemSsaVariable v) { result = v.(SignDef).getSign() }
+
+/** Gets a possible sign for `e`. */
+cached
+Sign semExprSign(SemExpr e) {
+  exists(Sign s | s = e.(SignExpr).getSign() |
+    if
+      getTrackedType(e) instanceof SemUnsignedIntegerType and
+      s = TNeg() and
+      not Specific::ignoreTypeRestrictions(e)
+    then result = TPos()
+    else result = s
+  )
+}
+
+/**
+ * Dummy predicate that holds for any sign. This is added to improve readability
+ * of cases where the sign is unrestricted.
+ */
+predicate semAnySign(Sign s) { any() }
+
+/** Holds if `e` can be positive and cannot be negative. */
+predicate semPositive(SemExpr e) {
+  semExprSign(e) = TPos() and
+  not semExprSign(e) = TNeg()
+}
+
+/** Holds if `e` can be negative and cannot be positive. */
+predicate semNegative(SemExpr e) {
+  semExprSign(e) = TNeg() and
+  not semExprSign(e) = TPos()
+}
+
+/** Holds if `e` is strictly positive. */
+predicate semStrictlyPositive(SemExpr e) {
+  semExprSign(e) = TPos() and
+  not semExprSign(e) = TNeg() and
+  not semExprSign(e) = TZero()
+}
+
+/** Holds if `e` is strictly negative. */
+predicate semStrictlyNegative(SemExpr e) {
+  semExprSign(e) = TNeg() and
+  not semExprSign(e) = TPos() and
+  not semExprSign(e) = TZero()
 }