Ruby: update crate versions

.codeqlmanifest.json

@@ -1,25 +1,11 @@
-{
+{ "provide": [ "ruby/.codeqlmanifest.json",
-    "provide": [
+                "*/ql/src/qlpack.yml",
-        "*/ql/src/qlpack.yml",
+               "*/ql/lib/qlpack.yml",
-        "*/ql/lib/qlpack.yml",
+               "*/ql/test/qlpack.yml",
-        "*/ql/test/qlpack.yml",
+               "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
-        "*/ql/examples/qlpack.yml",
+               "*/ql/examples/qlpack.yml",
-        "*/upgrades/qlpack.yml",
+               "*/upgrades/qlpack.yml",
-        "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
+               "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
-        "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
+               "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
-        "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
+               "misc/legacy-support/*/qlpack.yml",
-        "misc/legacy-support/*/qlpack.yml",
+               "misc/suite-helpers/qlpack.yml" ] }
-        "misc/suite-helpers/qlpack.yml",
-        "ruby/extractor-pack/codeql-extractor.yml",
-        "ruby/ql/consistency-queries/qlpack.yml",
-        "ql/ql/consistency-queries/qlpack.yml",
-        "ql/extractor-pack/codeql-extractor.yml"
-    ],
-    "versionPolicies": {
-      "default": {
-        "requireChangeNotes": true,
-        "committedPrereleaseSuffix": "dev",
-        "committedVersion": "nextPatchRelease"
-      }
-    }
-}

.github/actions/fetch-codeql/action.yml

@@ -8,7 +8,7 @@ runs:
       run: |
         LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
         gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
-        unzip -q -d "${RUNNER_TEMP}" codeql-linux64.zip
+        unzip -q codeql-linux64.zip
-        echo "${RUNNER_TEMP}/codeql" >> "${GITHUB_PATH}"
+        echo "${{ github.workspace }}/codeql" >> $GITHUB_PATH
       env:
         GITHUB_TOKEN: ${{ github.token }}

.github/labeler.yml

@@ -26,6 +26,3 @@ documentation:
   - "**/*.qhelp"
   - "**/*.md"
   - docs/**/*
-
-"QL-for-QL": 
-  - ql/**/*

.github/workflows/check-change-note.yml

@@ -7,7 +7,6 @@ on:
       - "*/ql/src/**/*.ql"
       - "*/ql/src/**/*.qll"
       - "!**/experimental/**"
-      - "!ql/**"
 
 jobs:
   check-change-note:

.github/workflows/post-pr-comment.yml

@@ -1,31 +0,0 @@
-name: Post pull-request comment
-on:
-  workflow_run:
-    workflows: ["Query help preview"]
-    types:
-      - completed
-
-permissions:
-  pull-requests: write
-
-jobs:
-  post_comment:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download artifact
-        run: gh run download "${WORKFLOW_RUN_ID}" --repo "${GITHUB_REPOSITORY}" --name "comment"
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-          WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
-      - run: |
-          PR="$(grep -o '^[0-9]\+$' pr.txt)"
-          PR_HEAD_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/pulls/${PR}" --jq .head.sha)"
-          # Check that the pull-request head SHA matches the head SHA of the workflow run
-          if [ "${WORKFLOW_RUN_HEAD_SHA}" != "${PR_HEAD_SHA}" ]; then
-            echo "PR head SHA ${PR_HEAD_SHA} does not match workflow_run event SHA ${WORKFLOW_RUN_HEAD_SHA}. Stopping." 1>&2
-            exit 1
-          fi
-          gh pr comment "${PR}" --repo "${GITHUB_REPOSITORY}" -F comment.txt
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-          WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_commit.id }}

.github/workflows/qhelp-pr-preview.yml

@@ -1,13 +1,10 @@
 name: Query help preview
 
-permissions:
-  contents: read
-
 on:
   pull_request:
     branches:
       - main
-      - "rc/*"
+      - 'rc/*'
     paths:
       - "ruby/**/*.qhelp"
 
@@ -15,49 +12,28 @@ jobs:
   qhelp:
     runs-on: ubuntu-latest
     steps:
-      - run: echo "${{  github.event.number }}" > pr.txt
-      - uses: actions/upload-artifact@v2
-        with:
-          name: comment
-          path: pr.txt
-          retention-days: 1
       - uses: actions/checkout@v2
         with:
           fetch-depth: 2
-          persist-credentials: false
-      - uses: ./.github/actions/fetch-codeql
       - name: Determine changed files
         id: changes
         run: |
-          (git diff -z --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep -z '.qhelp$' | grep -z -v '.inc.qhelp';
+          echo -n "::set-output name=qhelp_files::"
-           git diff -z --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep -z '.inc.qhelp$' | xargs --null -rn1 basename | xargs --null -rn1 git grep -z -l) |
+          (git diff --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep .qhelp$ | grep -v .inc.qhelp;
-           grep -z '.qhelp$' | grep -z -v '^-' | sort -z -u > "${RUNNER_TEMP}/paths.txt"
+           git diff --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep .inc.qhelp$ | xargs -d '\n' -rn1 basename | xargs -d '\n' -rn1 git grep -l) |
+           sort -u | xargs -d '\n' -n1 printf "'%s' "
+
+      - uses: ./.github/actions/fetch-codeql
 
       - name: QHelp preview
+        if: ${{ steps.changes.outputs.qhelp_files }}
         run: |
-          EXIT_CODE=0
+          ( echo "QHelp previews:";
-          echo "QHelp previews:" > comment.txt
+          for path in ${{ steps.changes.outputs.qhelp_files }} ; do
-          while read -r -d $'\0' path; do
-            if [ ! -f "${path}" ]; then
-               exit 1
-            fi
             echo "<details> <summary>${path}</summary>"
             echo
-            codeql generate query-help --format=markdown -- "./${path}" 2> errors.txt || EXIT_CODE="$?"
+            codeql generate query-help --format=markdown ${path}
-            if [ -s errors.txt ]; then
-               echo "# errors/warnings:"
-               echo '```'
-               cat errors.txt
-               cat errors.txt 1>&2
-               echo '```'
-            fi
             echo "</details>"
-          done < "${RUNNER_TEMP}/paths.txt" >> comment.txt
+          done) | gh pr comment "${{ github.event.pull_request.number }}" -F -
-          exit "${EXIT_CODE}"
+        env:
-
+          GITHUB_TOKEN: ${{ github.token }}
-      - if: always()
-        uses: actions/upload-artifact@v2
-        with:
-          name: comment
-          path: comment.txt
-          retention-days: 1

.github/workflows/ql-for-ql-build.yml

@@ -1,192 +0,0 @@
-name: Run QL for QL
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  queries:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
-        with:
-          languages: javascript # does not matter
-      - name: Get CodeQL version
-        id: get-codeql-version
-        run: |
-          echo "::set-output name=version::$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)"
-        shell: bash
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Cache queries
-        id: cache-queries
-        uses: actions/cache@v2
-        with:
-          path: ${{ runner.temp }}/query-pack.zip
-          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
-      - name: Build query pack
-        if: steps.cache-queries.outputs.cache-hit != 'true'
-        run: |
-          cd ql/ql/src
-          "${CODEQL}" pack create
-          cd .codeql/pack/codeql/ql-all/0.0.0
-          zip "${PACKZIP}" -r .
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-          PACKZIP: ${{ runner.temp }}/query-pack.zip
-      - name: Upload query pack
-        uses: actions/upload-artifact@v2
-        with:
-          name: query-pack-zip
-          path: ${{ runner.temp }}/query-pack.zip
-
-  extractors:
-    strategy:
-      fail-fast: false
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Cache entire extractor
-        id: cache-extractor
-        uses: actions/cache@v2
-        with:
-          path: |
-            ql/target/release/ql-autobuilder
-            ql/target/release/ql-autobuilder.exe
-            ql/target/release/ql-extractor
-            ql/target/release/ql-extractor.exe
-          key: ${{ runner.os }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
-      - name: Cache cargo
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
-      - name: Check formatting
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; cargo fmt --all -- --check
-      - name: Build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; cargo build --verbose
-      - name: Run tests
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; cargo test --verbose
-      - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; cargo build --release
-      - name: Generate dbscheme
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v2
-        with:
-          name: extractor-ubuntu-latest
-          path: |
-            ql/target/release/ql-autobuilder
-            ql/target/release/ql-autobuilder.exe
-            ql/target/release/ql-extractor
-            ql/target/release/ql-extractor.exe
-          retention-days: 1
-  package:
-    runs-on: ubuntu-latest
-
-    needs:
-      - extractors
-      - queries
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
-        with:
-          name: query-pack-zip
-          path: query-pack-zip
-      - uses: actions/download-artifact@v2
-        with:
-          name: extractor-ubuntu-latest
-          path: linux64
-      - run: |
-          unzip query-pack-zip/*.zip -d pack
-          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats pack/
-          mkdir -p pack/tools/linux64
-          if [[ -f linux64/ql-autobuilder ]]; then
-            cp linux64/ql-autobuilder pack/tools/linux64/autobuilder
-            chmod +x pack/tools/linux64/autobuilder
-          fi
-          if [[ -f linux64/ql-extractor ]]; then
-            cp linux64/ql-extractor pack/tools/linux64/extractor
-            chmod +x pack/tools/linux64/extractor
-          fi
-          cd pack
-          zip -rq ../codeql-ql.zip .
-      - uses: actions/upload-artifact@v2
-        with:
-          name: codeql-ql-pack
-          path: codeql-ql.zip
-          retention-days: 1
-  analyze:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        folder: [cpp, csharp, java, javascript, python, ql, ruby]
-
-    needs:
-      - package
-
-    steps:
-      - name: Download pack
-        uses: actions/download-artifact@v2
-        with:
-          name: codeql-ql-pack
-          path: ${{ runner.temp }}/codeql-ql-pack-artifact
-
-      - name: Prepare pack
-        run: |
-          unzip "${PACK_ARTIFACT}/*.zip" -d "${PACK}"
-        env:
-          PACK_ARTIFACT: ${{ runner.temp }}/codeql-ql-pack-artifact
-          PACK: ${{ runner.temp }}/pack
-      - name: Hack codeql-action options
-        run: |
-          JSON=$(jq -nc --arg pack "${PACK}" '.resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
-          echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
-        env:
-          PACK: ${{ runner.temp }}/pack
-
-      - name: Checkout repository
-        uses: actions/checkout@v2
-      - name: Create CodeQL config file
-        run: |
-          echo "paths:" > ${CONF}
-          echo "  - ${FOLDER}" >> ${CONF}
-          echo "paths-ignore:" >> ${CONF}
-          echo "  - ql/ql/test" >> ${CONF}
-          echo "Config file: "
-          cat ${CONF}
-        env: 
-          CONF: ./ql-for-ql-config.yml
-          FOLDER: ${{ matrix.folder }}
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@erik-krogh/ql
-        with:
-          languages: ql
-          db-location: ${{ runner.temp }}/db
-          config-file: ./ql-for-ql-config.yml
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@erik-krogh/ql
-        with: 
-          category: "ql-for-ql-${{ matrix.folder }}"
-

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -1,84 +0,0 @@
-name: Collect database stats for QL for QL
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - ql/ql/src/ql.dbscheme
-  pull_request:
-    branches: [main]
-    paths:
-      - ql/ql/src/ql.dbscheme
-  workflow_dispatch:
-
-jobs:
-  measure:
-    env:
-      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
-    strategy:
-      matrix:
-        repo: 
-          - github/codeql
-          - github/codeql-go
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
-        with:
-          languages: javascript # does not matter
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
-      - name: Build Extractor
-        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v2
-        with:
-          repository: ${{ matrix.repo }}
-          path: ${{ github.workspace }}/repo
-      - name: Create database
-        run: |
-          "${CODEQL}" database create \
-            --search-path "ql/extractor-pack" \
-            --threads 4 \
-            --language ql --source-root "${{ github.workspace }}/repo" \
-            "${{ runner.temp }}/database"
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Measure database
-        run: |
-          mkdir -p "stats/${{ matrix.repo }}"
-          "${CODEQL}" dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ql"
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - uses: actions/upload-artifact@v2
-        with:
-          name: measurements
-          path: stats
-          retention-days: 1
-
-  merge:
-    runs-on: ubuntu-latest
-    needs: measure
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
-        with:
-          name: measurements
-          path: stats
-      - run: |
-          python -m pip install --user lxml
-          find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ql/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
-      - uses: actions/upload-artifact@v2
-        with:
-          name: ql.dbscheme.stats
-          path: ql/ql/src/ql.dbscheme.stats

.github/workflows/ql-for-ql-tests.yml

@@ -1,52 +0,0 @@
-name: Run QL for QL Tests
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - "ql/**"
-  pull_request:
-    branches: [main]
-    paths:
-      - "ql/**"
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  qltest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
-        with:
-          languages: javascript # does not matter
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
-      - name: Build extractor
-        run: |
-          cd ql;
-          codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
-          env "PATH=$PATH:$codeqlpath" ./create-extractor-pack.sh
-      - name: Run QL tests
-        run: | 
-          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Check QL formatting
-        run: | 
-          find ql/ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Check QL compilation
-        run: | 
-          "${CODEQL}" query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}/ql/extractor-pack" "ql/ql/src" "ql/ql/examples"
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}

.github/workflows/ruby-build.yml

@@ -3,18 +3,16 @@ name: "Ruby: Build"
 on:
   push:
     paths:
-      - "ruby/**"
+      - 'ruby/**'
-      - .github/workflows/ruby-build.yml
     branches:
       - main
-      - "rc/*"
+      - 'rc/*'
   pull_request:
     paths:
-      - "ruby/**"
+      - 'ruby/**'
-      - .github/workflows/ruby-build.yml
     branches:
       - main
-      - "rc/*"
+      - 'rc/*'
   workflow_dispatch:
     inputs:
       tag:
@@ -102,6 +100,16 @@ jobs:
           PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
           codeql/codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
           (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
+      - name: Compile with previous CodeQL versions
+        run: |
+          for version in  $(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | sort --version-sort | tail -3 | head -2); do
+            rm -f codeql-linux64.zip
+            gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$version"
+            rm -rf codeql; unzip -q codeql-linux64.zip
+            codeql/codeql query compile target/packs/*
+          done
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
       - uses: actions/upload-artifact@v2
         with:
           name: codeql-ruby-queries

.github/workflows/ruby-dataset-measure.yml

@@ -4,17 +4,15 @@ on:
   push:
     branches:
       - main
-      - "rc/*"
+      - 'rc/*'
     paths:
       - ruby/ql/lib/ruby.dbscheme
-      - .github/workflows/ruby-dataset-measure.yml
   pull_request:
     branches:
       - main
-      - "rc/*"
+      - 'rc/*'
     paths:
       - ruby/ql/lib/ruby.dbscheme
-      - .github/workflows/ruby-dataset-measure.yml
   workflow_dispatch:
 
 jobs:
@@ -24,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        repo: [rails/rails, discourse/discourse, spree/spree, ruby/ruby]
+        repo: [rails/rails, discourse/discourse, spree/spree]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -41,7 +39,7 @@ jobs:
       - name: Create database
         run: |
           codeql database create \
-            --search-path "${{ github.workspace }}/ruby/extractor-pack" \
+            --search-path "${{ github.workspace }}/ruby" \
             --threads 4 \
             --language ruby --source-root "${{ github.workspace }}/repo" \
             "${{ runner.temp }}/database"

.github/workflows/ruby-qltest.yml

@@ -3,18 +3,16 @@ name: "Ruby: Run QL Tests"
 on:
   push:
     paths:
-      - "ruby/**"
+      - 'ruby/**'
-      - .github/workflows/ruby-qltest.yml
     branches:
       - main
-      - "rc/*"
+      - 'rc/*'
   pull_request:
     paths:
-      - "ruby/**"
+      - 'ruby/**'
-      - .github/workflows/ruby-qltest.yml
     branches:
       - main
-      - "rc/*"
+      - 'rc/*'
 
 env:
   CARGO_TERM_COLOR: always
@@ -32,19 +30,19 @@ jobs:
       - uses: ./ruby/actions/create-extractor-pack
       - name: Run QL tests
         run: |
-          codeql test run --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test
+          codeql test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ruby" --additional-packs "${{ github.workspace }}"  --consistency-queries ql/consistency-queries ql/test
         env:
           GITHUB_TOKEN: ${{ github.token }}
       - name: Check QL formatting
         run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
       - name: Check QL compilation
         run: |
-          codeql query compile --check-only --threads=4 --warnings=error "ql/src" "ql/examples"
+          codeql query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}/ruby" --additional-packs "${{ github.workspace }}" "ql/src" "ql/examples"
         env:
           GITHUB_TOKEN: ${{ github.token }}
       - name: Check DB upgrade scripts
         run: |
           echo >empty.trap
           codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
-          codeql dataset upgrade testdb --additional-packs ql/lib
+          codeql dataset upgrade testdb --additional-packs ql/lib/upgrades
           diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme

.gitignore

@@ -27,6 +27,3 @@ csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
 
 # Avoid committing cached package components
 .codeql
-
-# Compiled class file
-*.class

CODEOWNERS

@@ -25,6 +25,3 @@
 /docs/codeql-for-visual-studio-code/ @github/codeql-vscode-reviewers
 /docs/ql-language-reference/ @github/codeql-frontend-reviewers
 /docs/query-*-style-guide.md @github/codeql-analysis-reviewers
-
-# QL for QL reviewers
-/ql/ @erik-krogh @tausbn

CONTRIBUTING.md

@@ -11,14 +11,13 @@ If you have an idea for a query that you would like to share with other CodeQL u
 
 1. **Directory structure**
 
-    There are six language-specific query directories in this repository:
+    There are five language-specific query directories in this repository:
 
       * C/C++: `cpp/ql/src`
       * C#: `csharp/ql/src`
       * Java: `java/ql/src`
       * JavaScript: `javascript/ql/src`
       * Python: `python/ql/src`
-      * Ruby: `ruby/ql/src`
 
     Each language-specific directory contains further subdirectories that group queries based on their `@tags` or purpose.
     - Experimental queries and libraries are stored in the `experimental` subdirectory within each language-specific directory in the [CodeQL repository](https://github.com/github/codeql). For example, experimental Java queries and libraries are stored in `java/ql/src/experimental` and any corresponding tests in `java/ql/test/experimental`.

config/identical-files.json

@@ -449,27 +449,19 @@
     "csharp/ql/lib/semmle/code/csharp/controlflow/internal/pressa/SsaImplCommon.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/basessa/SsaImplCommon.qll",
     "csharp/ql/lib/semmle/code/cil/internal/SsaImplCommon.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/SsaImplCommon.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/SsaImplCommon.qll"
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaImplCommon.qll"
   ],
-  "CryptoAlgorithms Python/JS/Ruby": [
+  "CryptoAlgorithms Python/JS": [
     "javascript/ql/lib/semmle/javascript/security/CryptoAlgorithms.qll",
-    "python/ql/lib/semmle/python/concepts/CryptoAlgorithms.qll",
+    "python/ql/lib/semmle/python/concepts/CryptoAlgorithms.qll"
-    "ruby/ql/lib/codeql/ruby/security/CryptoAlgorithms.qll"
-  ],
-  "CryptoAlgorithmNames Python/JS/Ruby": [
-    "javascript/ql/lib/semmle/javascript/security/internal/CryptoAlgorithmNames.qll",
-    "python/ql/lib/semmle/python/concepts/internal/CryptoAlgorithmNames.qll",
-    "ruby/ql/lib/codeql/ruby/security/internal/CryptoAlgorithmNames.qll"
   ],
   "SensitiveDataHeuristics Python/JS": [
     "javascript/ql/lib/semmle/javascript/security/internal/SensitiveDataHeuristics.qll",
     "python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll"
   ],
-  "ReDoS Util Python/JS/Ruby": [
+  "ReDoS Util Python/JS": [
     "javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll",
-    "python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll",
+    "python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll"
-    "ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll"
   ],
   "ReDoS Exponential Python/JS": [
     "javascript/ql/lib/semmle/javascript/security/performance/ExponentialBackTracking.qll",
@@ -478,12 +470,7 @@
   "ReDoS Polynomial Python/JS": [
     "javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll",
     "python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll",
-    "ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll"
+    "ruby/ql/lib/codeql/ruby/regexp/SuperlinearBackTracking.qll"
-  ],
-  "BadTagFilterQuery Python/JS/Ruby": [
-    "javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll",
-    "python/ql/lib/semmle/python/security/BadTagFilterQuery.qll",
-    "ruby/ql/lib/codeql/ruby/security/BadTagFilterQuery.qll"
   ],
   "CFG": [
     "csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll",

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -17,7 +17,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Build" Version="16.11.0" />
+    <PackageReference Include="Microsoft.Build" Version="16.9.0" />
   </ItemGroup>
 
   <ItemGroup>

cpp/old-change-notes/2021-10-07-cleartext-transmission.md

@@ -1,2 +0,0 @@
-lgtm,codescanning
-* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query has been improved, reducing the number of false positive results when encryption is present.

cpp/ql/lib/CHANGELOG.md

@@ -1,13 +0,0 @@
-## 0.0.7
-
-## 0.0.6
-
-## 0.0.5
-
-## 0.0.4
-
-### New Features
-
-* The QL library `semmle.code.cpp.commons.Exclusions` now contains a predicate
-  `isFromSystemMacroDefinition` for identifying code that originates from a
-  macro outside the project being analyzed.

cpp/ql/lib/DefaultOptions.qll

@@ -73,7 +73,7 @@ class Options extends string {
    *   __assume(0);
    * ```
    * (note that in this case if the hint is wrong and the expression is reached at
-   * runtime, the program's behavior is undefined)
+   * runtime, the program's behaviour is undefined)
    */
   predicate exprExits(Expr e) {
     e.(AssumeExpr).getChild(0).(CompileTimeConstantInt).getIntValue() = 0 or

cpp/ql/lib/Options.qll

@@ -50,7 +50,7 @@ class CustomOptions extends Options {
    *   __assume(0);
    * ```
    * (note that in this case if the hint is wrong and the expression is reached at
-   * runtime, the program's behavior is undefined)
+   * runtime, the program's behaviour is undefined)
    */
   override predicate exprExits(Expr e) { Options.super.exprExits(e) }
 

cpp/ql/lib/change-notes/released/0.0.4.md

@@ -1,7 +0,0 @@
-## 0.0.4
-
-### New Features
-
-* The QL library `semmle.code.cpp.commons.Exclusions` now contains a predicate
-  `isFromSystemMacroDefinition` for identifying code that originates from a
-  macro outside the project being analyzed.

cpp/ql/lib/change-notes/released/0.0.5.md

@@ -1 +0,0 @@
-## 0.0.5

cpp/ql/lib/change-notes/released/0.0.6.md

@@ -1 +0,0 @@
-## 0.0.6

cpp/ql/lib/change-notes/released/0.0.7.md

@@ -1 +0,0 @@
-## 0.0.7

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +0,0 @@
----
-lastReleaseVersion: 0.0.7

cpp/ql/lib/experimental/semmle/code/cpp/models/interfaces/SimpleRangeAnalysisDefinition.qll

@@ -37,7 +37,7 @@ abstract class SimpleRangeAnalysisDefinition extends RangeSsaDefinition {
    * dependencies. Without this information, range analysis might work for
    * simple cases but will go into infinite loops on complex code.
    *
-   * For example, when modeling the definition by reference in a call to an
+   * For example, when modelling the definition by reference in a call to an
    * overloaded `operator=`, written as `v = e`, the definition of `(this, v)`
    * depends on `e`.
    */

cpp/ql/lib/experimental/semmle/code/cpp/rangeanalysis/InBoundsPointerDeref.qll

@@ -5,7 +5,7 @@
  * `Instruction` level), and then using the array length analysis and the range
  * analysis together to prove that some of these pointer dereferences are safe.
  *
- * The analysis is soundy, i.e. it is sound if no undefined behavior is present
+ * The analysis is soundy, i.e. it is sound if no undefined behaviour is present
  * in the program.
  * Furthermore, it crucially depends on the soundiness of the range analysis and
  * the array length analysis.

cpp/ql/lib/qlpack.yml

@@ -1,8 +1,7 @@
 name: codeql/cpp-all
-version: 0.0.7
+version: 0.0.2
-groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp
 library: true
 dependencies:
-  codeql/cpp-upgrades: ^0.0.3
+  codeql/cpp-upgrades: 0.0.2

cpp/ql/lib/semmle/code/cpp/Class.qll

@@ -237,7 +237,7 @@ class Class extends UserType {
     exists(ClassDerivation cd | cd.getBaseClass() = base |
       result =
         this.accessOfBaseMemberMulti(cd.getDerivedClass(),
-          fieldInBase.accessInDirectDerived(cd.getASpecifier()))
+          fieldInBase.accessInDirectDerived(cd.getASpecifier().(AccessSpecifier)))
     )
   }
 
@@ -261,7 +261,8 @@ class Class extends UserType {
    * includes the case of `base` = `this`.
    */
   AccessSpecifier accessOfBaseMember(Declaration member) {
-    result = this.accessOfBaseMember(member.getDeclaringType(), member.getASpecifier())
+    result =
+      this.accessOfBaseMember(member.getDeclaringType(), member.getASpecifier().(AccessSpecifier))
   }
 
   /**
@@ -318,7 +319,7 @@ class Class extends UserType {
     exists(Type t | t = this.getAFieldSubobjectType().getUnspecifiedType() |
       // Note: Overload resolution is not implemented -- all copy
       // constructors are considered equal.
-      this.cannotAccessCopyConstructorOnAny(t)
+      this.cannotAccessCopyConstructorOnAny(t.(Class))
     )
     or
     // - T has direct or virtual base class that cannot be copied (has deleted,
@@ -391,7 +392,7 @@ class Class extends UserType {
     exists(Type t | t = this.getAFieldSubobjectType().getUnspecifiedType() |
       // Note: Overload resolution is not implemented -- all copy assignment
       // operators are considered equal.
-      this.cannotAccessCopyAssignmentOperatorOnAny(t)
+      this.cannotAccessCopyAssignmentOperatorOnAny(t.(Class))
     )
     or
     exists(Class c | c = this.getADirectOrVirtualBase() |

cpp/ql/lib/semmle/code/cpp/Declaration.qll

@@ -490,7 +490,8 @@ class AccessHolder extends Declaration, TAccessHolder {
    */
   pragma[inline]
   predicate canAccessMember(Declaration member, Class derived) {
-    this.couldAccessMember(member.getDeclaringType(), member.getASpecifier(), derived)
+    this.couldAccessMember(member.getDeclaringType(), member.getASpecifier().(AccessSpecifier),
+      derived)
   }
 
   /**

cpp/ql/lib/semmle/code/cpp/commons/Dependency.qll

@@ -275,7 +275,7 @@ private predicate dependsOnDeclarationEntry(Element src, DeclarationEntry dest)
     dependsOnTransitive(src, mid) and
     not mid instanceof Type and
     not mid instanceof EnumConstant and
-    getDeclarationEntries(mid, dest) and
+    getDeclarationEntries(mid, dest.(DeclarationEntry)) and
     not dest instanceof TypeDeclarationEntry
   )
   or
@@ -283,9 +283,9 @@ private predicate dependsOnDeclarationEntry(Element src, DeclarationEntry dest)
     // dependency from a Type / Variable / Function use -> any (visible) definition
     dependsOnTransitive(src, mid) and
     not mid instanceof EnumConstant and
-    getDeclarationEntries(mid, dest) and
+    getDeclarationEntries(mid, dest.(DeclarationEntry)) and
     // must be definition
-    dest.isDefinition()
+    dest.(DeclarationEntry).isDefinition()
   )
 }
 
@@ -307,7 +307,7 @@ private predicate dependsOnFull(DependsSource src, Symbol dest, int category) {
     // dependency from a Variable / Function use -> non-visible definition (link time)
     dependsOnTransitive(src, mid) and
     not mid instanceof EnumConstant and
-    getDeclarationEntries(mid, dest) and
+    getDeclarationEntries(mid, dest.(DeclarationEntry)) and
     not dest instanceof TypeDeclarationEntry and
     // must be definition
     dest.(DeclarationEntry).isDefinition() and

cpp/ql/lib/semmle/code/cpp/commons/Exclusions.qll

@@ -81,8 +81,8 @@ predicate functionContainsPreprocCode(Function f) {
 }
 
 /**
- * Holds if `e` is completely or partially from a macro invocation `mi`, as
+ * Holds if `e` is completely or partially from a macro definition, as opposed
- * opposed to being passed in as an argument.
+ * to being passed in as an argument.
  *
  * In the following example, the call to `f` is from a macro definition,
  * while `y`, `+`, `1`, and `;` are not. This assumes that no identifier apart
@@ -93,8 +93,8 @@ predicate functionContainsPreprocCode(Function f) {
  *   M(y + 1);
  * ```
  */
-private predicate isFromMacroInvocation(Element e, MacroInvocation mi) {
+predicate isFromMacroDefinition(Element e) {
-  exists(Location eLocation, Location miLocation |
+  exists(MacroInvocation mi, Location eLocation, Location miLocation |
     mi.getAnExpandedElement() = e and
     eLocation = e.getLocation() and
     miLocation = mi.getLocation() and
@@ -109,36 +109,3 @@ private predicate isFromMacroInvocation(Element e, MacroInvocation mi) {
     eLocation.getEndColumn() >= miLocation.getEndColumn()
   )
 }
-
-/**
- * Holds if `e` is completely or partially from a macro definition, as opposed
- * to being passed in as an argument.
- *
- * In the following example, the call to `f` is from a macro definition,
- * while `y`, `+`, `1`, and `;` are not. This assumes that no identifier apart
- * from `M` refers to a macro.
- * ```
- * #define M(x) f(x)
- * ...
- *   M(y + 1);
- * ```
- */
-predicate isFromMacroDefinition(Element e) { isFromMacroInvocation(e, _) }
-
-/**
- * Holds if `e` is completely or partially from a _system macro_ definition, as
- * opposed to being passed in as an argument. A system macro is a macro whose
- * definition is outside the source directory of the database.
- *
- * If the system macro is invoked through a non-system macro, then this
- * predicate does not hold.
- *
- * See also `isFromMacroDefinition`.
- */
-predicate isFromSystemMacroDefinition(Element e) {
-  exists(MacroInvocation mi |
-    isFromMacroInvocation(e, mi) and
-    // Has no relative path in the database, meaning it's a system file.
-    not exists(mi.getMacro().getFile().getRelativePath())
-  )
-}

cpp/ql/lib/semmle/code/cpp/commons/NullTermination.qll

@@ -3,33 +3,17 @@ private import semmle.code.cpp.models.interfaces.ArrayFunction
 private import semmle.code.cpp.models.implementations.Strcat
 import semmle.code.cpp.dataflow.DataFlow
 
-/**
+private predicate mayAddNullTerminatorHelper(Expr e, VariableAccess va, Expr e0) {
- * Holds if the expression `e` assigns something including `va` to a
+  exists(StackVariable v0, Expr val |
- * stack variable `v0`.
+    exprDefinition(v0, e, val) and
- */
+    val.getAChild*() = va and
-private predicate mayAddNullTerminatorHelper(Expr e, VariableAccess va, StackVariable v0) {
+    mayAddNullTerminator(e0, v0.getAnAccess())
-  exists(Expr val |
-    exprDefinition(v0, e, val) and // `e` is `v0 := val`
-    val.getAChild*() = va
-  )
-}
-
-bindingset[n1, n2]
-private predicate controlFlowNodeSuccessorTransitive(ControlFlowNode n1, ControlFlowNode n2) {
-  exists(BasicBlock bb1, int pos1, BasicBlock bb2, int pos2 |
-    pragma[only_bind_into](bb1).getNode(pos1) = n1 and
-    pragma[only_bind_into](bb2).getNode(pos2) = n2 and
-    (
-      bb1 = bb2 and pos1 < pos2
-      or
-      bb1.getASuccessor+() = bb2
-    )
   )
 }
 
 /**
- * Holds if the expression `e` may add a null terminator to the string
+ * Holds if the expression `e` may add a null terminator to the string in
- * accessed by `va`.
+ * variable `v`.
  */
 predicate mayAddNullTerminator(Expr e, VariableAccess va) {
   // Assignment: dereferencing or array access
@@ -46,10 +30,14 @@ predicate mayAddNullTerminator(Expr e, VariableAccess va) {
   )
   or
   // Assignment to another stack variable
-  exists(StackVariable v0, Expr e0 |
+  exists(Expr e0, BasicBlock bb, int pos, BasicBlock bb0, int pos0 |
-    mayAddNullTerminatorHelper(e, va, v0) and
+    mayAddNullTerminatorHelper(e, va, e0) and
-    mayAddNullTerminator(pragma[only_bind_into](e0), pragma[only_bind_into](v0.getAnAccess())) and
+    bb.getNode(pos) = e and
-    controlFlowNodeSuccessorTransitive(e, e0)
+    bb0.getNode(pos0) = e0
+  |
+    bb = bb0 and pos < pos0
+    or
+    bb.getASuccessor+() = bb0
   )
   or
   // Assignment to non-stack variable
@@ -131,9 +119,14 @@ predicate variableMustBeNullTerminated(VariableAccess va) {
       variableMustBeNullTerminated(use) and
       // Simplified: check that `p` may not be null terminated on *any*
       // path to `use` (including the one found via `parameterUsePair`)
-      not exists(Expr e |
+      not exists(Expr e, BasicBlock bb1, int pos1, BasicBlock bb2, int pos2 |
-        mayAddNullTerminator(pragma[only_bind_into](e), p.getAnAccess()) and
+        mayAddNullTerminator(e, p.getAnAccess()) and
-        controlFlowNodeSuccessorTransitive(e, use)
+        bb1.getNode(pos1) = e and
+        bb2.getNode(pos2) = use
+      |
+        bb1 = bb2 and pos1 < pos2
+        or
+        bb1.getASuccessor+() = bb2
       )
     )
   )

cpp/ql/lib/semmle/code/cpp/commons/Printf.qll

@@ -6,85 +6,6 @@ import semmle.code.cpp.Type
 import semmle.code.cpp.commons.CommonType
 import semmle.code.cpp.commons.StringAnalysis
 import semmle.code.cpp.models.interfaces.FormattingFunction
-private import semmle.code.cpp.rangeanalysis.SimpleRangeAnalysis
-private import semmle.code.cpp.rangeanalysis.RangeAnalysisUtils
-
-private newtype TBufferWriteEstimationReason =
-  TNoSpecifiedEstimateReason() or
-  TTypeBoundsAnalysis() or
-  TValueFlowAnalysis()
-
-/**
- * A reason for a specific buffer write size estimate.
- */
-abstract class BufferWriteEstimationReason extends TBufferWriteEstimationReason {
-  /**
-   * Returns the name of the concrete class.
-   */
-  abstract string toString();
-
-  /**
-   * Returns a human readable representation of this reason.
-   */
-  abstract string getDescription();
-
-  /**
-   * Combine estimate reasons. Used to give a reason for the size of a format string
-   * conversion given reasons coming from its individual specifiers.
-   */
-  abstract BufferWriteEstimationReason combineWith(BufferWriteEstimationReason other);
-}
-
-/**
- * No particular reason given. This is currently used for backward compatibility so that
- * classes derived from BufferWrite and overriding `getMaxData/0` still work with the
- * queries as intended.
- */
-class NoSpecifiedEstimateReason extends BufferWriteEstimationReason, TNoSpecifiedEstimateReason {
-  override string toString() { result = "NoSpecifiedEstimateReason" }
-
-  override string getDescription() { result = "no reason specified" }
-
-  override BufferWriteEstimationReason combineWith(BufferWriteEstimationReason other) {
-    // this reason should not be used in format specifiers, so it should not be combined
-    // with other reasons
-    none()
-  }
-}
-
-/**
- * The estimation comes from rough bounds just based on the type (e.g.
- * `0 <= x < 2^32` for an unsigned 32 bit integer).
- */
-class TypeBoundsAnalysis extends BufferWriteEstimationReason, TTypeBoundsAnalysis {
-  override string toString() { result = "TypeBoundsAnalysis" }
-
-  override string getDescription() { result = "based on type bounds" }
-
-  override BufferWriteEstimationReason combineWith(BufferWriteEstimationReason other) {
-    other != TNoSpecifiedEstimateReason() and result = TTypeBoundsAnalysis()
-  }
-}
-
-/**
- * The estimation comes from non trivial bounds found via actual flow analysis.
- * For example
- * ```
- * unsigned u = x;
- * if (u < 1000) {
- *    //...  <- estimation done here based on u
- * }
- * ```
- */
-class ValueFlowAnalysis extends BufferWriteEstimationReason, TValueFlowAnalysis {
-  override string toString() { result = "ValueFlowAnalysis" }
-
-  override string getDescription() { result = "based on flow analysis of value bounds" }
-
-  override BufferWriteEstimationReason combineWith(BufferWriteEstimationReason other) {
-    other != TNoSpecifiedEstimateReason() and result = other
-  }
-}
 
 class PrintfFormatAttribute extends FormatAttribute {
   PrintfFormatAttribute() { this.getArchetype() = ["printf", "__printf__"] }
@@ -254,7 +175,9 @@ class FormattingFunctionCall extends Expr {
   /**
    * Gets the index at which the format string occurs in the argument list.
    */
-  int getFormatParameterIndex() { result = this.getTarget().getFormatParameterIndex() }
+  int getFormatParameterIndex() {
+    result = this.getTarget().(FormattingFunction).getFormatParameterIndex()
+  }
 
   /**
    * Gets the format expression used in this call.
@@ -268,7 +191,7 @@ class FormattingFunctionCall extends Expr {
     exists(int i |
       result = this.getArgument(i) and
       n >= 0 and
-      n = i - this.getTarget().getFirstFormatArgumentIndex()
+      n = i - this.getTarget().(FormattingFunction).getFirstFormatArgumentIndex()
     )
   }
 
@@ -328,7 +251,7 @@ class FormattingFunctionCall extends Expr {
   int getNumFormatArgument() {
     result = count(this.getFormatArgument(_)) and
     // format arguments must be known
-    exists(this.getTarget().getFirstFormatArgumentIndex())
+    exists(this.getTarget().(FormattingFunction).getFirstFormatArgumentIndex())
   }
 
   /**
@@ -347,18 +270,6 @@ class FormattingFunctionCall extends Expr {
   }
 }
 
-/**
- * Gets the number of digits required to represent the integer represented by `f`.
- *
- * `f` is assumed to be nonnegative.
- */
-bindingset[f]
-private int lengthInBase10(float f) {
-  f = 0 and result = 1
-  or
-  result = f.log10().floor() + 1
-}
-
 /**
  * A class to represent format strings that occur as arguments to invocations of formatting functions.
  */
@@ -378,27 +289,35 @@ class FormatLiteral extends Literal {
    * a `char *` (either way, `%S` will have the opposite meaning).
    * DEPRECATED: Use getDefaultCharType() instead.
    */
-  deprecated predicate isWideCharDefault() { this.getUse().getTarget().isWideCharDefault() }
+  deprecated predicate isWideCharDefault() {
+    this.getUse().getTarget().(FormattingFunction).isWideCharDefault()
+  }
 
   /**
    * Gets the default character type expected for `%s` by this format literal.  Typically
    * `char` or `wchar_t`.
    */
-  Type getDefaultCharType() { result = this.getUse().getTarget().getDefaultCharType() }
+  Type getDefaultCharType() {
+    result = this.getUse().getTarget().(FormattingFunction).getDefaultCharType()
+  }
 
   /**
    * Gets the non-default character type expected for `%S` by this format literal.  Typically
    * `wchar_t` or `char`.  On some snapshots there may be multiple results where we can't tell
    * which is correct for a particular function.
    */
-  Type getNonDefaultCharType() { result = this.getUse().getTarget().getNonDefaultCharType() }
+  Type getNonDefaultCharType() {
+    result = this.getUse().getTarget().(FormattingFunction).getNonDefaultCharType()
+  }
 
   /**
    * Gets the wide character type for this format literal.  This is usually `wchar_t`.  On some
    * snapshots there may be multiple results where we can't tell which is correct for a
    * particular function.
    */
-  Type getWideCharType() { result = this.getUse().getTarget().getWideCharType() }
+  Type getWideCharType() {
+    result = this.getUse().getTarget().(FormattingFunction).getWideCharType()
+  }
 
   /**
    * Holds if this `FormatLiteral` is in a context that supports
@@ -977,7 +896,7 @@ class FormatLiteral extends Literal {
     exists(string len, string conv |
       this.parseConvSpec(n, _, _, _, _, _, len, conv) and
       (len != "l" and len != "w" and len != "h") and
-      this.getUse().getTarget().getFormatCharType().getSize() > 1 and // wide function
+      this.getUse().getTarget().(FormattingFunction).getFormatCharType().getSize() > 1 and // wide function
       (
         conv = "c" and
         result = this.getNonDefaultCharType()
@@ -1067,14 +986,7 @@ class FormatLiteral extends Literal {
    * conversion specifier of this format string; has no result if this cannot
    * be determined.
    */
-  int getMaxConvertedLength(int n) { result = max(getMaxConvertedLength(n, _)) }
+  int getMaxConvertedLength(int n) {
-
-  /**
-   * Gets the maximum length of the string that can be produced by the nth
-   * conversion specifier of this format string, specifying the estimation reason;
-   * has no result if this cannot be determined.
-   */
-  int getMaxConvertedLength(int n, BufferWriteEstimationReason reason) {
     exists(int len |
       (
         (
@@ -1086,12 +998,10 @@ class FormatLiteral extends Literal {
       ) and
       (
         this.getConversionChar(n) = "%" and
-        len = 1 and
+        len = 1
-        reason = TValueFlowAnalysis()
         or
         this.getConversionChar(n).toLowerCase() = "c" and
-        len = 1 and
+        len = 1 // e.g. 'a'
-        reason = TValueFlowAnalysis() // e.g. 'a'
         or
         this.getConversionChar(n).toLowerCase() = "f" and
         exists(int dot, int afterdot |
@@ -1105,8 +1015,7 @@ class FormatLiteral extends Literal {
             afterdot = 6
           ) and
           len = 1 + 309 + dot + afterdot
-        ) and
+        ) // e.g. -1e308="-100000"...
-        reason = TTypeBoundsAnalysis() // e.g. -1e308="-100000"...
         or
         this.getConversionChar(n).toLowerCase() = "e" and
         exists(int dot, int afterdot |
@@ -1120,8 +1029,7 @@ class FormatLiteral extends Literal {
             afterdot = 6
           ) and
           len = 1 + 1 + dot + afterdot + 1 + 1 + 3
-        ) and
+        ) // -1e308="-1.000000e+308"
-        reason = TTypeBoundsAnalysis() // -1e308="-1.000000e+308"
         or
         this.getConversionChar(n).toLowerCase() = "g" and
         exists(int dot, int afterdot |
@@ -1144,79 +1052,42 @@ class FormatLiteral extends Literal {
           //       (e.g. 123456, 0.000123456 are just OK)
           //       so case %f can be at most P characters + 4 zeroes, sign, dot = P + 6
           len = (afterdot.maximum(1) + 6).maximum(1 + 1 + dot + afterdot + 1 + 1 + 3)
-        ) and
+        ) // (e.g. "-1.59203e-319")
-        reason = TTypeBoundsAnalysis() // (e.g. "-1.59203e-319")
         or
         this.getConversionChar(n).toLowerCase() = ["d", "i"] and
         // e.g. -2^31 = "-2147483648"
-        exists(float typeBasedBound, float valueBasedBound |
+        exists(int sizeBits |
-          // The first case handles length sub-specifiers
+          sizeBits =
-          // Subtract one in the exponent because one bit is for the sign.
+            min(int bits |
-          // Add 1 to account for the possible sign in the output.
+              bits = this.getIntegralDisplayType(n).getSize() * 8
-          typeBasedBound =
+              or
-            1 + lengthInBase10(2.pow(this.getIntegralDisplayType(n).getSize() * 8 - 1)) and
+              exists(IntegralType t |
-          // The second case uses range analysis to deduce a length that's shorter than the length
+                t = this.getUse().getConversionArgument(n).getType().getUnderlyingType()
-          // of the number -2^31.
+              |
-          exists(Expr arg, float lower, float upper, float typeLower, float typeUpper |
+                t.isSigned() and bits = t.getSize() * 8
-            arg = this.getUse().getConversionArgument(n) and
+              )
-            lower = lowerBound(arg.getFullyConverted()) and
+            ) and
-            upper = upperBound(arg.getFullyConverted()) and
+          len = 1 + ((sizeBits - 1) / 10.0.log2()).ceil()
-            typeLower = exprMinVal(arg.getFullyConverted()) and
+          // this calculation is as %u (below) only we take out the sign bit (- 1) and allow a whole
-            typeUpper = exprMaxVal(arg.getFullyConverted())
+          // character for it to be expressed as '-'.
-          |
-            valueBasedBound =
-              max(int cand |
-                // Include the sign bit in the length if it can be negative
-                (
-                  if lower < 0
-                  then cand = 1 + lengthInBase10(lower.abs())
-                  else cand = lengthInBase10(lower)
-                )
-                or
-                (
-                  if upper < 0
-                  then cand = 1 + lengthInBase10(upper.abs())
-                  else cand = lengthInBase10(upper)
-                )
-              ) and
-            (
-              if lower > typeLower or upper < typeUpper
-              then reason = TValueFlowAnalysis()
-              else reason = TTypeBoundsAnalysis()
-            )
-          ) and
-          len = valueBasedBound.minimum(typeBasedBound)
         )
         or
         this.getConversionChar(n).toLowerCase() = "u" and
         // e.g. 2^32 - 1 = "4294967295"
-        exists(float typeBasedBound, float valueBasedBound |
+        exists(int sizeBits |
-          // The first case handles length sub-specifiers
+          sizeBits =
-          typeBasedBound = lengthInBase10(2.pow(this.getIntegralDisplayType(n).getSize() * 8) - 1) and
+            min(int bits |
-          // The second case uses range analysis to deduce a length that's shorter than
+              bits = this.getIntegralDisplayType(n).getSize() * 8
-          // the length of the number 2^31 - 1.
+              or
-          exists(Expr arg, float lower, float upper, float typeLower, float typeUpper |
+              exists(IntegralType t |
-            arg = this.getUse().getConversionArgument(n) and
+                t = this.getUse().getConversionArgument(n).getType().getUnderlyingType()
-            lower = lowerBound(arg.getFullyConverted()) and
+              |
-            upper = upperBound(arg.getFullyConverted()) and
+                t.isUnsigned() and bits = t.getSize() * 8
-            typeLower = exprMinVal(arg.getFullyConverted()) and
+              )
-            typeUpper = exprMaxVal(arg.getFullyConverted())
+            ) and
-          |
+          len = (sizeBits / 10.0.log2()).ceil()
-            valueBasedBound =
+          // convert the size from bits to decimal characters, and round up as you can't have
-              lengthInBase10(max(float cand |
+          // fractional characters (10.0.log2() is the number of bits expressed per decimal character)
-                  // If lower can be negative we use `(unsigned)-1` as the candidate value.
-                  lower < 0 and
-                  cand = 2.pow(any(IntType t | t.isUnsigned()).getSize() * 8)
-                  or
-                  cand = upper
-                )) and
-            (
-              if lower > typeLower or upper < typeUpper
-              then reason = TValueFlowAnalysis()
-              else reason = TTypeBoundsAnalysis()
-            )
-          ) and
-          len = valueBasedBound.minimum(typeBasedBound)
         )
         or
         this.getConversionChar(n).toLowerCase() = "x" and
@@ -1236,8 +1107,7 @@ class FormatLiteral extends Literal {
           (
             if this.hasAlternateFlag(n) then len = 2 + baseLen else len = baseLen // "0x"
           )
-        ) and
+        )
-        reason = TTypeBoundsAnalysis()
         or
         this.getConversionChar(n).toLowerCase() = "p" and
         exists(PointerType ptrType, int baseLen |
@@ -1246,8 +1116,7 @@ class FormatLiteral extends Literal {
           (
             if this.hasAlternateFlag(n) then len = 2 + baseLen else len = baseLen // "0x"
           )
-        ) and
+        )
-        reason = TValueFlowAnalysis()
         or
         this.getConversionChar(n).toLowerCase() = "o" and
         // e.g. 2^32 - 1 = "37777777777"
@@ -1266,16 +1135,14 @@ class FormatLiteral extends Literal {
           (
             if this.hasAlternateFlag(n) then len = 1 + baseLen else len = baseLen // "0"
           )
-        ) and
+        )
-        reason = TTypeBoundsAnalysis()
         or
         this.getConversionChar(n).toLowerCase() = "s" and
         len =
           min(int v |
             v = this.getPrecision(n) or
             v = this.getUse().getFormatArgument(n).(AnalysedString).getMaxLength() - 1 // (don't count null terminator)
-          ) and
+          )
-        reason = TValueFlowAnalysis()
       )
     )
   }
@@ -1287,19 +1154,10 @@ class FormatLiteral extends Literal {
    * determining whether a buffer overflow is caused by long float to string
    * conversions.
    */
-  int getMaxConvertedLengthLimited(int n) { result = max(getMaxConvertedLengthLimited(n, _)) }
+  int getMaxConvertedLengthLimited(int n) {
-
-  /**
-   * Gets the maximum length of the string that can be produced by the nth
-   * conversion specifier of this format string, specifying the reason for the
-   * estimation, except that float to string conversions are assumed to be 8
-   * characters.  This is helpful for determining whether a buffer overflow is
-   * caused by long float to string conversions.
-   */
-  int getMaxConvertedLengthLimited(int n, BufferWriteEstimationReason reason) {
     if this.getConversionChar(n).toLowerCase() = "f"
-    then result = this.getMaxConvertedLength(n, reason).minimum(8)
+    then result = this.getMaxConvertedLength(n).minimum(8)
-    else result = this.getMaxConvertedLength(n, reason)
+    else result = this.getMaxConvertedLength(n)
   }
 
   /**
@@ -1339,35 +1197,29 @@ class FormatLiteral extends Literal {
     )
   }
 
-  private int getMaxConvertedLengthAfter(int n, BufferWriteEstimationReason reason) {
+  private int getMaxConvertedLengthAfter(int n) {
     if n = this.getNumConvSpec()
-    then result = this.getConstantSuffix().length() + 1 and reason = TValueFlowAnalysis()
+    then result = this.getConstantSuffix().length() + 1
     else
-      exists(BufferWriteEstimationReason headReason, BufferWriteEstimationReason tailReason |
+      result =
-        result =
+        this.getConstantPart(n).length() + this.getMaxConvertedLength(n) +
-          this.getConstantPart(n).length() + this.getMaxConvertedLength(n, headReason) +
+          this.getMaxConvertedLengthAfter(n + 1)
-            this.getMaxConvertedLengthAfter(n + 1, tailReason) and
-        reason = headReason.combineWith(tailReason)
-      )
   }
 
-  private int getMaxConvertedLengthAfterLimited(int n, BufferWriteEstimationReason reason) {
+  private int getMaxConvertedLengthAfterLimited(int n) {
     if n = this.getNumConvSpec()
-    then result = this.getConstantSuffix().length() + 1 and reason = TValueFlowAnalysis()
+    then result = this.getConstantSuffix().length() + 1
     else
-      exists(BufferWriteEstimationReason headReason, BufferWriteEstimationReason tailReason |
+      result =
-        result =
+        this.getConstantPart(n).length() + this.getMaxConvertedLengthLimited(n) +
-          this.getConstantPart(n).length() + this.getMaxConvertedLengthLimited(n, headReason) +
+          this.getMaxConvertedLengthAfterLimited(n + 1)
-            this.getMaxConvertedLengthAfterLimited(n + 1, tailReason) and
-        reason = headReason.combineWith(tailReason)
-      )
   }
 
   /**
    * Gets the maximum length of the string that can be produced by this format
    * string.  Has no result if this cannot be determined.
    */
-  int getMaxConvertedLength() { result = this.getMaxConvertedLengthAfter(0, _) }
+  int getMaxConvertedLength() { result = this.getMaxConvertedLengthAfter(0) }
 
   /**
    * Gets the maximum length of the string that can be produced by this format
@@ -1375,24 +1227,5 @@ class FormatLiteral extends Literal {
    * characters.  This is helpful for determining whether a buffer overflow
    * is caused by long float to string conversions.
    */
-  int getMaxConvertedLengthLimited() { result = this.getMaxConvertedLengthAfterLimited(0, _) }
+  int getMaxConvertedLengthLimited() { result = this.getMaxConvertedLengthAfterLimited(0) }
-
-  /**
-   * Gets the maximum length of the string that can be produced by this format
-   * string, specifying the reason for the estimate. Has no result if no estimate
-   * can be found.
-   */
-  int getMaxConvertedLengthWithReason(BufferWriteEstimationReason reason) {
-    result = this.getMaxConvertedLengthAfter(0, reason)
-  }
-
-  /**
-   * Gets the maximum length of the string that can be produced by this format
-   * string, specifying the reason for the estimate, except that float to string
-   * conversions are assumed to be 8 characters.  This is helpful for determining
-   * whether a buffer overflow is caused by long float to string conversions.
-   */
-  int getMaxConvertedLengthLimitedWithReason(BufferWriteEstimationReason reason) {
-    result = this.getMaxConvertedLengthAfterLimited(0, reason)
-  }
 }

cpp/ql/lib/semmle/code/cpp/controlflow/DefinitionsAndUses.qll

@@ -25,7 +25,7 @@ predicate definitionUsePair(SemanticStackVariable var, Expr def, Expr use) {
  * Holds if the definition `def` of some stack variable can reach `node`, which
  * is a definition or use, without crossing definitions of the same variable.
  */
-predicate definitionReaches(Expr def, Expr node) { def.(Def).reaches(true, _, node) }
+predicate definitionReaches(Expr def, Expr node) { def.(Def).reaches(true, _, node.(DefOrUse)) }
 
 private predicate hasAddressOfAccess(SemanticStackVariable var) {
   var.getAnAccess().isAddressOfAccessNonConst()

cpp/ql/lib/semmle/code/cpp/controlflow/SSA.qll

@@ -62,7 +62,7 @@ class SsaDefinition extends ControlFlowNodeBase {
   BasicBlock getBasicBlock() { result.contains(this.getDefinition()) }
 
   /** Holds if this definition is a phi node for variable `v`. */
-  predicate isPhiNode(StackVariable v) { exists(StandardSSA x | x.phi_node(v, this)) }
+  predicate isPhiNode(StackVariable v) { exists(StandardSSA x | x.phi_node(v, this.(BasicBlock))) }
 
   /** Gets the location of this definition. */
   Location getLocation() { result = this.(ControlFlowNode).getLocation() }

cpp/ql/lib/semmle/code/cpp/controlflow/SSAUtils.qll

@@ -153,11 +153,9 @@ library class SSAHelper extends int {
    * Modern Compiler Implementation by Andrew Appel.
    */
   private predicate frontier_phi_node(StackVariable v, BasicBlock b) {
-    exists(BasicBlock x |
+    exists(BasicBlock x | dominanceFrontier(x, b) and ssa_defn_rec(v, x)) and
-      dominanceFrontier(x, b) and ssa_defn_rec(pragma[only_bind_into](v), pragma[only_bind_into](x))
-    ) and
     /* We can also eliminate those nodes where the variable is not live on any incoming edge */
-    live_at_start_of_bb(pragma[only_bind_into](v), b)
+    live_at_start_of_bb(v, b)
   }
 
   private predicate ssa_defn_rec(StackVariable v, BasicBlock b) {
@@ -294,7 +292,7 @@ library class SSAHelper extends int {
    */
   cached
   string toString(ControlFlowNode node, StackVariable v) {
-    if phi_node(v, node)
+    if phi_node(v, node.(BasicBlock))
     then result = "SSA phi(" + v.getName() + ")"
     else (
       ssa_defn(v, node, _, _) and result = "SSA def(" + v.getName() + ")"

cpp/ql/lib/semmle/code/cpp/controlflow/internal/CFG.qll

@@ -231,7 +231,7 @@ private class PostOrderInitializer extends Initializer {
       or
       this.getDeclaration() = for.getRangeVariable()
       or
-      this.getDeclaration() = for.getBeginEndDeclaration().getADeclaration()
+      this.getDeclaration() = for.getBeginEndDeclaration().(DeclStmt).getADeclaration()
     )
   }
 }
@@ -1143,7 +1143,7 @@ private class ExceptionSource extends Node {
       this.reachesParent(mid) and
       not mid = any(TryStmt try).getStmt() and
       not mid = any(MicrosoftTryStmt try).getStmt() and
-      parent = mid.getParentNode()
+      parent = mid.(Node).getParentNode()
     )
   }
 

cpp/ql/lib/semmle/code/cpp/controlflow/internal/ConstantExprs.qll

@@ -484,7 +484,7 @@ library class ExprEvaluator extends int {
       this.interestingInternal(e, req, true) and
       (
         result = req.(CompileTimeConstantInt).getIntValue() or
-        result = this.getCompoundValue(e, req)
+        result = this.getCompoundValue(e, req.(CompileTimeVariableExpr))
       ) and
       (
         req.getUnderlyingType().(IntegralType).isSigned() or
@@ -611,7 +611,7 @@ library class ExprEvaluator extends int {
       or
       exists(AssignExpr req | req = val | result = this.getValueInternal(e, req.getRValue()))
       or
-      result = this.getVariableValue(e, val)
+      result = this.getVariableValue(e, val.(VariableAccess))
       or
       exists(FunctionCall call | call = val and not callWithMultipleTargets(call) |
         result = this.getFunctionValue(call.getTarget())
@@ -626,9 +626,9 @@ library class ExprEvaluator extends int {
       // All assignments must have the same int value
       result =
         unique(Expr value |
-          value = v.getAnAssignedValue() and not this.ignoreVariableAssignment(e, v, value)
+          value = v.getAnAssignedValue() and not ignoreVariableAssignment(e, v, value)
         |
-          this.getValueInternalNonSubExpr(value)
+          getValueInternalNonSubExpr(value)
         )
     )
   }
@@ -663,7 +663,7 @@ library class ExprEvaluator extends int {
     this.interestingInternal(_, req, false) and
     (
       result = req.(CompileTimeConstantInt).getIntValue() or
-      result = this.getCompoundValueNonSubExpr(req)
+      result = this.getCompoundValueNonSubExpr(req.(CompileTimeVariableExpr))
     ) and
     (
       req.getUnderlyingType().(IntegralType).isSigned() or
@@ -787,7 +787,7 @@ library class ExprEvaluator extends int {
       or
       exists(AssignExpr req | req = val | result = this.getValueInternalNonSubExpr(req.getRValue()))
       or
-      result = this.getVariableValueNonSubExpr(val)
+      result = this.getVariableValueNonSubExpr(val.(VariableAccess))
       or
       exists(FunctionCall call | call = val and not callWithMultipleTargets(call) |
         result = this.getFunctionValue(call.getTarget())

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowDispatch.qll

@@ -1,6 +1,4 @@
 private import cpp
-private import semmle.code.cpp.dataflow.internal.DataFlowPrivate
-private import semmle.code.cpp.dataflow.internal.DataFlowUtil
 
 /**
  * Gets a function that might be called by `call`.
@@ -65,17 +63,3 @@ predicate mayBenefitFromCallContext(Call call, Function f) { none() }
  * restricted to those `call`s for which a context might make a difference.
  */
 Function viableImplInCallContext(Call call, Call ctx) { none() }
-
-/** A parameter position represented by an integer. */
-class ParameterPosition extends int {
-  ParameterPosition() { any(ParameterNode p).isParameterOf(_, this) }
-}
-
-/** An argument position represented by an integer. */
-class ArgumentPosition extends int {
-  ArgumentPosition() { any(ArgumentNode a).argumentOf(_, this) }
-}
-
-/** Holds if arguments at position `apos` match parameters at position `ppos`. */
-pragma[inline]
-predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) { ppos = apos }

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll

@@ -2,42 +2,6 @@ private import DataFlowImplSpecific::Private
 private import DataFlowImplSpecific::Public
 import Cached
 
-module DataFlowImplCommonPublic {
-  private newtype TFlowFeature =
-    TFeatureHasSourceCallContext() or
-    TFeatureHasSinkCallContext() or
-    TFeatureEqualSourceSinkCallContext()
-
-  /** A flow configuration feature for use in `Configuration::getAFeature()`. */
-  class FlowFeature extends TFlowFeature {
-    string toString() { none() }
-  }
-
-  /**
-   * A flow configuration feature that implies that sources have some existing
-   * call context.
-   */
-  class FeatureHasSourceCallContext extends FlowFeature, TFeatureHasSourceCallContext {
-    override string toString() { result = "FeatureHasSourceCallContext" }
-  }
-
-  /**
-   * A flow configuration feature that implies that sinks have some existing
-   * call context.
-   */
-  class FeatureHasSinkCallContext extends FlowFeature, TFeatureHasSinkCallContext {
-    override string toString() { result = "FeatureHasSinkCallContext" }
-  }
-
-  /**
-   * A flow configuration feature that implies that source-sink pairs have some
-   * shared existing call context.
-   */
-  class FeatureEqualSourceSinkCallContext extends FlowFeature, TFeatureEqualSourceSinkCallContext {
-    override string toString() { result = "FeatureEqualSourceSinkCallContext" }
-  }
-}
-
 /**
  * The cost limits for the `AccessPathFront` to `AccessPathApprox` expansion.
  *
@@ -62,18 +26,6 @@ predicate accessPathCostLimits(int apLimit, int tupleLimit) {
   tupleLimit = 1000
 }
 
-/**
- * Holds if `arg` is an argument of `call` with an argument position that matches
- * parameter position `ppos`.
- */
-pragma[noinline]
-predicate argumentPositionMatch(DataFlowCall call, ArgNode arg, ParameterPosition ppos) {
-  exists(ArgumentPosition apos |
-    arg.argumentOf(call, apos) and
-    parameterMatch(ppos, apos)
-  )
-}
-
 /**
  * Provides a simple data-flow analysis for resolving lambda calls. The analysis
  * currently excludes read-steps, store-steps, and flow-through.
@@ -83,27 +35,25 @@ predicate argumentPositionMatch(DataFlowCall call, ArgNode arg, ParameterPositio
  * calls. For this reason, we cannot reuse the code from `DataFlowImpl.qll` directly.
  */
 private module LambdaFlow {
-  pragma[noinline]
+  private predicate viableParamNonLambda(DataFlowCall call, int i, ParamNode p) {
-  private predicate viableParamNonLambda(DataFlowCall call, ParameterPosition ppos, ParamNode p) {
+    p.isParameterOf(viableCallable(call), i)
-    p.isParameterOf(viableCallable(call), ppos)
   }
 
-  pragma[noinline]
+  private predicate viableParamLambda(DataFlowCall call, int i, ParamNode p) {
-  private predicate viableParamLambda(DataFlowCall call, ParameterPosition ppos, ParamNode p) {
+    p.isParameterOf(viableCallableLambda(call, _), i)
-    p.isParameterOf(viableCallableLambda(call, _), ppos)
   }
 
   private predicate viableParamArgNonLambda(DataFlowCall call, ParamNode p, ArgNode arg) {
-    exists(ParameterPosition ppos |
+    exists(int i |
-      viableParamNonLambda(call, ppos, p) and
+      viableParamNonLambda(call, i, p) and
-      argumentPositionMatch(call, arg, ppos)
+      arg.argumentOf(call, i)
     )
   }
 
   private predicate viableParamArgLambda(DataFlowCall call, ParamNode p, ArgNode arg) {
-    exists(ParameterPosition ppos |
+    exists(int i |
-      viableParamLambda(call, ppos, p) and
+      viableParamLambda(call, i, p) and
-      argumentPositionMatch(call, arg, ppos)
+      arg.argumentOf(call, i)
     )
   }
 
@@ -301,7 +251,7 @@ private module Cached {
   predicate forceCachingInSameStage() { any() }
 
   cached
-  predicate nodeEnclosingCallable(Node n, DataFlowCallable c) { c = nodeGetEnclosingCallable(n) }
+  predicate nodeEnclosingCallable(Node n, DataFlowCallable c) { c = n.getEnclosingCallable() }
 
   cached
   predicate callEnclosingCallable(DataFlowCall call, DataFlowCallable c) {
@@ -336,7 +286,7 @@ private module Cached {
     or
     exists(ArgNode arg |
       result.(PostUpdateNode).getPreUpdateNode() = arg and
-      arg.argumentOf(call, k.(ParamUpdateReturnKind).getAMatchingArgumentPosition())
+      arg.argumentOf(call, k.(ParamUpdateReturnKind).getPosition())
     )
   }
 
@@ -344,7 +294,7 @@ private module Cached {
   predicate returnNodeExt(Node n, ReturnKindExt k) {
     k = TValueReturn(n.(ReturnNode).getKind())
     or
-    exists(ParamNode p, ParameterPosition pos |
+    exists(ParamNode p, int pos |
       parameterValueFlowsToPreUpdate(p, n) and
       p.isParameterOf(_, pos) and
       k = TParamUpdate(pos)
@@ -366,13 +316,13 @@ private module Cached {
   }
 
   cached
-  predicate parameterNode(Node p, DataFlowCallable c, ParameterPosition pos) {
+  predicate parameterNode(Node n, DataFlowCallable c, int i) {
-    isParameterNode(p, c, pos)
+    n.(ParameterNode).isParameterOf(c, i)
   }
 
   cached
-  predicate argumentNode(Node n, DataFlowCall call, ArgumentPosition pos) {
+  predicate argumentNode(Node n, DataFlowCall call, int pos) {
-    isArgumentNode(n, call, pos)
+    n.(ArgumentNode).argumentOf(call, pos)
   }
 
   /**
@@ -390,12 +340,12 @@ private module Cached {
   }
 
   /**
-   * Holds if `p` is the parameter of a viable dispatch target of `call`,
+   * Holds if `p` is the `i`th parameter of a viable dispatch target of `call`.
-   * and `p` has position `ppos`.
+   * The instance parameter is considered to have index `-1`.
    */
   pragma[nomagic]
-  private predicate viableParam(DataFlowCall call, ParameterPosition ppos, ParamNode p) {
+  private predicate viableParam(DataFlowCall call, int i, ParamNode p) {
-    p.isParameterOf(viableCallableExt(call), ppos)
+    p.isParameterOf(viableCallableExt(call), i)
   }
 
   /**
@@ -404,9 +354,9 @@ private module Cached {
    */
   cached
   predicate viableParamArg(DataFlowCall call, ParamNode p, ArgNode arg) {
-    exists(ParameterPosition ppos |
+    exists(int i |
-      viableParam(call, ppos, p) and
+      viableParam(call, i, p) and
-      argumentPositionMatch(call, arg, ppos) and
+      arg.argumentOf(call, i) and
       compatibleTypes(getNodeDataFlowType(arg), getNodeDataFlowType(p))
     )
   }
@@ -878,7 +828,7 @@ private module Cached {
   cached
   newtype TReturnKindExt =
     TValueReturn(ReturnKind kind) or
-    TParamUpdate(ParameterPosition pos) { exists(ParamNode p | p.isParameterOf(_, pos)) }
+    TParamUpdate(int pos) { exists(ParamNode p | p.isParameterOf(_, pos)) }
 
   cached
   newtype TBooleanOption =
@@ -1070,9 +1020,9 @@ class ParamNode extends Node {
 
   /**
    * Holds if this node is the parameter of callable `c` at the specified
-   * position.
+   * (zero-based) position.
    */
-  predicate isParameterOf(DataFlowCallable c, ParameterPosition pos) { parameterNode(this, c, pos) }
+  predicate isParameterOf(DataFlowCallable c, int i) { parameterNode(this, c, i) }
 }
 
 /** A data-flow node that represents a call argument. */
@@ -1080,9 +1030,7 @@ class ArgNode extends Node {
   ArgNode() { argumentNode(this, _, _) }
 
   /** Holds if this argument occurs at the given position in the given call. */
-  final predicate argumentOf(DataFlowCall call, ArgumentPosition pos) {
+  final predicate argumentOf(DataFlowCall call, int pos) { argumentNode(this, call, pos) }
-    argumentNode(this, call, pos)
-  }
 }
 
 /**
@@ -1128,14 +1076,11 @@ class ValueReturnKind extends ReturnKindExt, TValueReturn {
 }
 
 class ParamUpdateReturnKind extends ReturnKindExt, TParamUpdate {
-  private ParameterPosition pos;
+  private int pos;
 
   ParamUpdateReturnKind() { this = TParamUpdate(pos) }
 
-  ParameterPosition getPosition() { result = pos }
+  int getPosition() { result = pos }
-
-  pragma[nomagic]
-  ArgumentPosition getAMatchingArgumentPosition() { parameterMatch(pos, result) }
 
   override string toString() { result = "param update " + pos }
 }

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplConsistency.qll

@@ -9,31 +9,6 @@ private import tainttracking1.TaintTrackingParameter::Private
 private import tainttracking1.TaintTrackingParameter::Public
 
 module Consistency {
-  private newtype TConsistencyConfiguration = MkConsistencyConfiguration()
-
-  /** A class for configuring the consistency queries. */
-  class ConsistencyConfiguration extends TConsistencyConfiguration {
-    string toString() { none() }
-
-    /** Holds if `n` should be excluded from the consistency test `uniqueEnclosingCallable`. */
-    predicate uniqueEnclosingCallableExclude(Node n) { none() }
-
-    /** Holds if `n` should be excluded from the consistency test `uniqueNodeLocation`. */
-    predicate uniqueNodeLocationExclude(Node n) { none() }
-
-    /** Holds if `n` should be excluded from the consistency test `missingLocation`. */
-    predicate missingLocationExclude(Node n) { none() }
-
-    /** Holds if `n` should be excluded from the consistency test `postWithInFlow`. */
-    predicate postWithInFlowExclude(Node n) { none() }
-
-    /** Holds if `n` should be excluded from the consistency test `argHasPostUpdate`. */
-    predicate argHasPostUpdateExclude(ArgumentNode n) { none() }
-
-    /** Holds if `n` should be excluded from the consistency test `reverseRead`. */
-    predicate reverseReadExclude(Node n) { none() }
-  }
-
   private class RelevantNode extends Node {
     RelevantNode() {
       this instanceof ArgumentNode or
@@ -56,9 +31,8 @@ module Consistency {
   query predicate uniqueEnclosingCallable(Node n, string msg) {
     exists(int c |
       n instanceof RelevantNode and
-      c = count(nodeGetEnclosingCallable(n)) and
+      c = count(n.getEnclosingCallable()) and
       c != 1 and
-      not any(ConsistencyConfiguration conf).uniqueEnclosingCallableExclude(n) and
       msg = "Node should have one enclosing callable but has " + c + "."
     )
   }
@@ -79,7 +53,6 @@ module Consistency {
           n.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
         ) and
       c != 1 and
-      not any(ConsistencyConfiguration conf).uniqueNodeLocationExclude(n) and
       msg = "Node should have one location but has " + c + "."
     )
   }
@@ -90,8 +63,7 @@ module Consistency {
         strictcount(Node n |
           not exists(string filepath, int startline, int startcolumn, int endline, int endcolumn |
             n.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
-          ) and
+          )
-          not any(ConsistencyConfiguration conf).missingLocationExclude(n)
         ) and
       msg = "Nodes without location: " + c
     )
@@ -113,13 +85,13 @@ module Consistency {
   }
 
   query predicate parameterCallable(ParameterNode p, string msg) {
-    exists(DataFlowCallable c | isParameterNode(p, c, _) and c != nodeGetEnclosingCallable(p)) and
+    exists(DataFlowCallable c | p.isParameterOf(c, _) and c != p.getEnclosingCallable()) and
     msg = "Callable mismatch for parameter."
   }
 
   query predicate localFlowIsLocal(Node n1, Node n2, string msg) {
     simpleLocalFlowStep(n1, n2) and
-    nodeGetEnclosingCallable(n1) != nodeGetEnclosingCallable(n2) and
+    n1.getEnclosingCallable() != n2.getEnclosingCallable() and
     msg = "Local flow step does not preserve enclosing callable."
   }
 
@@ -134,7 +106,7 @@ module Consistency {
   query predicate unreachableNodeCCtx(Node n, DataFlowCall call, string msg) {
     isUnreachableInCall(n, call) and
     exists(DataFlowCallable c |
-      c = nodeGetEnclosingCallable(n) and
+      c = n.getEnclosingCallable() and
       not viableCallable(call) = c
     ) and
     msg = "Call context for isUnreachableInCall is inconsistent with call graph."
@@ -148,7 +120,7 @@ module Consistency {
       n.(ArgumentNode).argumentOf(call, _) and
       msg = "ArgumentNode and call does not share enclosing callable."
     ) and
-    nodeGetEnclosingCallable(n) != call.getEnclosingCallable()
+    n.getEnclosingCallable() != call.getEnclosingCallable()
   }
 
   // This predicate helps the compiler forget that in some languages
@@ -179,7 +151,7 @@ module Consistency {
   }
 
   query predicate postIsInSameCallable(PostUpdateNode n, string msg) {
-    nodeGetEnclosingCallable(n) != nodeGetEnclosingCallable(n.getPreUpdateNode()) and
+    n.getEnclosingCallable() != n.getPreUpdateNode().getEnclosingCallable() and
     msg = "PostUpdateNode does not share callable with its pre-update node."
   }
 
@@ -187,13 +159,12 @@ module Consistency {
 
   query predicate reverseRead(Node n, string msg) {
     exists(Node n2 | readStep(n, _, n2) and hasPost(n2) and not hasPost(n)) and
-    not any(ConsistencyConfiguration conf).reverseReadExclude(n) and
     msg = "Origin of readStep is missing a PostUpdateNode."
   }
 
   query predicate argHasPostUpdate(ArgumentNode n, string msg) {
     not hasPost(n) and
-    not any(ConsistencyConfiguration c).argHasPostUpdateExclude(n) and
+    not isImmutableOrUnobservable(n) and
     msg = "ArgumentNode is missing PostUpdateNode."
   }
 
@@ -206,7 +177,6 @@ module Consistency {
     isPostUpdateNode(n) and
     not clearsContent(n, _) and
     simpleLocalFlowStep(_, n) and
-    not any(ConsistencyConfiguration c).postWithInFlowExclude(n) and
     msg = "PostUpdateNode should not be the target of local flow."
   }
 }

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowPrivate.qll

@@ -2,20 +2,6 @@ private import cpp
 private import DataFlowUtil
 private import DataFlowDispatch
 private import FlowVar
-private import DataFlowImplConsistency
-
-/** Gets the callable in which this node occurs. */
-DataFlowCallable nodeGetEnclosingCallable(Node n) { result = n.getEnclosingCallable() }
-
-/** Holds if `p` is a `ParameterNode` of `c` with position `pos`. */
-predicate isParameterNode(ParameterNode p, DataFlowCallable c, ParameterPosition pos) {
-  p.isParameterOf(c, pos)
-}
-
-/** Holds if `arg` is an `ArgumentNode` of `c` with position `pos`. */
-predicate isArgumentNode(ArgumentNode arg, DataFlowCall c, ArgumentPosition pos) {
-  arg.argumentOf(c, pos)
-}
 
 /** Gets the instance argument of a non-static call. */
 private Node getInstanceArgument(Call call) {
@@ -267,6 +253,27 @@ class Unit extends TUnit {
   string toString() { result = "unit" }
 }
 
+/**
+ * Holds if `n` does not require a `PostUpdateNode` as it either cannot be
+ * modified or its modification cannot be observed, for example if it is a
+ * freshly created object that is not saved in a variable.
+ *
+ * This predicate is only used for consistency checks.
+ */
+predicate isImmutableOrUnobservable(Node n) {
+  // Is the null pointer (or something that's not really a pointer)
+  exists(n.asExpr().getValue())
+  or
+  // Isn't a pointer or is a pointer to const
+  forall(DerivedType dt | dt = n.asExpr().getActualType() |
+    dt.getBaseType().isConst()
+    or
+    dt.getBaseType() instanceof RoutineType
+  )
+  // The above list of cases isn't exhaustive, but it narrows down the
+  // consistency alerts enough that most of them are interesting.
+}
+
 /** Holds if `n` should be hidden from path explanations. */
 predicate nodeIsHidden(Node n) { none() }
 
@@ -289,19 +296,3 @@ predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preserves
  * by default as a heuristic.
  */
 predicate allowParameterReturnInSelf(ParameterNode p) { none() }
-
-private class MyConsistencyConfiguration extends Consistency::ConsistencyConfiguration {
-  override predicate argHasPostUpdateExclude(ArgumentNode n) {
-    // Is the null pointer (or something that's not really a pointer)
-    exists(n.asExpr().getValue())
-    or
-    // Isn't a pointer or is a pointer to const
-    forall(DerivedType dt | dt = n.asExpr().getActualType() |
-      dt.getBaseType().isConst()
-      or
-      dt.getBaseType() instanceof RoutineType
-    )
-    // The above list of cases isn't exhaustive, but it narrows down the
-    // consistency alerts enough that most of them are interesting.
-  }
-}

cpp/ql/lib/semmle/code/cpp/exprs/Expr.qll

@@ -31,7 +31,7 @@ class Expr extends StmtParent, @expr {
   override Stmt getEnclosingStmt() {
     result = this.getParent().(Expr).getEnclosingStmt()
     or
-    result = this.getParent()
+    result = this.getParent().(Stmt)
     or
     exists(Expr other | result = other.getEnclosingStmt() and other.getConversion() = this)
     or

cpp/ql/lib/semmle/code/cpp/exprs/Lambda.qll

@@ -118,7 +118,7 @@ class LambdaCapture extends Locatable, @lambdacapture {
    * An identifier is captured by reference if:
    *   - It is explicitly captured by reference.
    *   - It is implicitly captured, and the lambda's default capture mode is by-reference.
-   *   - The identifier is "this". [Said behavior is dictated by the C++11 standard, but it
+   *   - The identifier is "this". [Said behaviour is dictated by the C++11 standard, but it
    *                                is actually "*this" being captured rather than "this".]
    */
   predicate isCapturedByReference() { lambda_capture(this, _, _, _, true, _, _) }