Merge pull request #10553 from github/8032-python-2-deprecation

Minor updates to reflect recommendations for Python support

.bazelrc

@@ -1,3 +1,3 @@
-build --repo_env=CC=clang --repo_env=CXX=clang++ --copt="-std=c++17"
+build --repo_env=CC=clang --repo_env=CXX=clang++ --cxxopt="-std=c++17"
 
 try-import %workspace%/local.bazelrc

.codeqlmanifest.json

@@ -1,27 +0,0 @@
-{
-    "provide": [
-        "*/ql/src/qlpack.yml",
-        "*/ql/lib/qlpack.yml",
-        "*/ql/test/qlpack.yml",
-        "*/ql/examples/qlpack.yml",
-        "*/ql/consistency-queries/qlpack.yml",
-        "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
-        "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
-        "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml",
-        "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
-        "csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
-        "csharp/ql/campaigns/Solorigate/src/qlpack.yml",
-        "csharp/ql/campaigns/Solorigate/test/qlpack.yml",
-        "misc/legacy-support/*/qlpack.yml",
-        "misc/suite-helpers/qlpack.yml",
-        "ruby/extractor-pack/codeql-extractor.yml",
-        "ql/extractor-pack/codeql-extractor.yml"
-    ],
-    "versionPolicies": {
-      "default": {
-        "requireChangeNotes": true,
-        "committedPrereleaseSuffix": "dev",
-        "committedVersion": "nextPatchRelease"
-      }
-    }
-}

.devcontainer/devcontainer.json

@@ -3,6 +3,8 @@
     "rust-lang.rust",
     "bungcip.better-toml",
     "github.vscode-codeql",
+    "hbenl.vscode-test-explorer",
+    "ms-vscode.test-adapter-converter",
     "slevesque.vscode-zipexplorer"
   ],
   "settings": {

.devcontainer/swift/Dockerfile

@@ -0,0 +1,9 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.236.0/containers/cpp/.devcontainer/base.Dockerfile
+
+# [Choice] Debian / Ubuntu version (use Debian 11, Ubuntu 18.04/22.04 on local arm64/Apple Silicon): debian-11, debian-10, ubuntu-22.04, ubuntu-20.04, ubuntu-18.04
+FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-ubuntu-22.04
+
+USER root
+ADD root.sh /tmp/root.sh
+ADD update-codeql.sh /usr/local/bin/update-codeql
+RUN bash /tmp/root.sh && rm /tmp/root.sh

.devcontainer/swift/devcontainer.json

@@ -0,0 +1,25 @@
+{
+	"extensions": [
+		"github.vscode-codeql",
+		"hbenl.vscode-test-explorer",
+		"ms-vscode.test-adapter-converter",
+		"slevesque.vscode-zipexplorer",
+		"ms-vscode.cpptools"
+	],
+	"settings": {
+		"files.watcherExclude": {
+			"**/target/**": true
+		},
+		"codeQL.runningQueries.memory": 2048
+	},
+	"build": {
+		"dockerfile": "Dockerfile",
+	},
+	"runArgs": [
+		"--cap-add=SYS_PTRACE",
+		"--security-opt",
+		"seccomp=unconfined"
+	],
+	"remoteUser": "vscode",
+	"onCreateCommand": ".devcontainer/swift/user.sh"
+}

.devcontainer/swift/root.sh

@@ -0,0 +1,22 @@
+set -xe
+
+BAZELISK_VERSION=v1.12.0
+BAZELISK_DOWNLOAD_SHA=6b0bcb2ea15bca16fffabe6fda75803440375354c085480fe361d2cbf32501db
+
+apt-get update
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y install --no-install-recommends \
+    zlib1g-dev \
+    uuid-dev \
+    python3-distutils \
+    python3-pip \
+    bash-completion
+
+# Install Bazel
+curl -fSsL -o /usr/local/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-amd64
+echo "${BAZELISK_DOWNLOAD_SHA} */usr/local/bin/bazelisk" | sha256sum --check -
+chmod 0755 /usr/local/bin/bazelisk
+ln -s bazelisk /usr/local/bin/bazel
+
+# install latest codeql
+update-codeql

.devcontainer/swift/update-codeql.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+
+URL=https://github.com/github/codeql-cli-binaries/releases
+LATEST_VERSION=$(curl -L -s -H 'Accept: application/json' $URL/latest | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/')
+CURRENT_VERSION=v$(codeql version 2>/dev/null | sed -ne 's/.*release \([0-9.]*\)\./\1/p')
+if [[ $CURRENT_VERSION != $LATEST_VERSION ]]; then
+  if [[ $UID != 0 ]]; then
+    echo "update required, please run this script with sudo:"
+    echo "  sudo $0"
+    exit 1
+  fi
+  ZIP=$(mktemp codeql.XXXX.zip)
+  curl -fSqL -o $ZIP $URL/download/$LATEST_VERSION/codeql-linux64.zip
+  unzip -q $ZIP -d /opt
+  rm $ZIP
+  ln -sf /opt/codeql/codeql /usr/local/bin/codeql
+  echo installed version $LATEST_VERSION
+else
+  echo current version $CURRENT_VERSION is up-to-date
+fi

.devcontainer/swift/user.sh

@@ -0,0 +1,13 @@
+set -xe
+
+# add the workspace to the codeql search path
+mkdir -p /home/vscode/.config/codeql
+echo "--search-path /workspaces/codeql" > /home/vscode/.config/codeql/config
+
+# create a swift extractor pack with the current state
+cd /workspaces/codeql
+bazel run swift/create-extractor-pack
+
+#install and set up pre-commit
+python3 -m pip install pre-commit --no-warn-script-location
+$HOME/.local/bin/pre-commit install

.gitattributes

@@ -39,6 +39,7 @@
 *.py text
 *.lua text
 *.expected text
+*.go text
 
 # Explicitly set a bunch of known extensions to binary, because Git < 2.10 will treat
 # `* text=auto eol=lf` as `* text eol=lf`
@@ -52,6 +53,14 @@
 java/ql/test/stubs/**/*.java linguist-generated=true
 java/ql/test/experimental/stubs/**/*.java linguist-generated=true
 
+# Force git not to modify line endings for go or html files under the go/ql directory
+go/ql/**/*.go -text
+go/ql/**/*.html -text
+# Force git not to modify line endings for go dbschemes
+go/*.dbscheme -text
+# Preserve unusual line ending from codeql-go merge
+go/extractor/opencsv/CSVReader.java -text
+
 # For some languages, upgrade script testing references really old dbscheme
 # files from legacy upgrades that have CRLF line endings. Since upgrade
 # resolution relies on object hashes, we must suppress line ending conversion

.github/actions/fetch-codeql/action.yml

@@ -3,22 +3,12 @@ description: Fetches the latest version of CodeQL
 runs:
   using: composite
   steps:
-    - name: Select platform - Linux
-      if: runner.os == 'Linux'
-      shell: bash
-      run: echo "GA_CODEQL_CLI_PLATFORM=linux64" >> $GITHUB_ENV
-
-    - name: Select platform - MacOS
-      if: runner.os == 'MacOS'
-      shell: bash
-      run: echo "GA_CODEQL_CLI_PLATFORM=osx64" >> $GITHUB_ENV
-
     - name: Fetch CodeQL
       shell: bash
       run: |
-        LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
-        gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-$GA_CODEQL_CLI_PLATFORM.zip "$LATEST"
-        unzip -q -d "${RUNNER_TEMP}" codeql-$GA_CODEQL_CLI_PLATFORM.zip
-        echo "${RUNNER_TEMP}/codeql" >> "${GITHUB_PATH}"
+        gh extension install github/gh-codeql
+        gh codeql set-channel nightly
+        gh codeql version
+        gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_PATH}"
       env:
         GITHUB_TOKEN: ${{ github.token }}

.github/labeler.yml

@@ -6,14 +6,23 @@
   - csharp/**/*
   - change-notes/**/*csharp*
 
+Go:
+  - go/**/*
+  - change-notes/**/*go.*
+
 Java:
-  - java/**/*
+  - any: [ 'java/**/*', '!java/kotlin-extractor/**/*', '!java/kotlin-explorer/**/*', '!java/ql/test/kotlin/**/*' ]
   - change-notes/**/*java.*
 
 JS:
   - any: [ 'javascript/**/*', '!javascript/ql/experimental/adaptivethreatmodeling/**/*' ]
   - change-notes/**/*javascript*
 
+Kotlin:
+  - java/kotlin-extractor/**/*
+  - java/kotlin-explorer/**/*
+  - java/ql/test/kotlin/**/*
+
 Python:
   - python/**/*
   - change-notes/**/*python*
@@ -21,7 +30,7 @@ Python:
 Ruby:
   - ruby/**/*
   - change-notes/**/*ruby*
-  
+
 Swift:
   - swift/**/*
   - change-notes/**/*swift*
@@ -31,5 +40,6 @@ documentation:
   - "**/*.md"
   - docs/**/*
 
-"QL-for-QL": 
+"QL-for-QL":
   - ql/**/*
+  - .github/workflows/ql-for-ql*

.github/problem-matchers/codeql-query-format.json

@@ -0,0 +1,14 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "codeql-query-format",
+            "pattern": [
+                {
+                    "regexp": "^((.*) would change by autoformatting\\.)$",
+                    "file": 2,
+                    "message": 1
+                }
+            ]
+        }
+    ]
+}

.github/problem-matchers/codeql-syntax-check.json

@@ -0,0 +1,17 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "codeql-syntax-check",
+            "pattern": [
+                {
+                    "regexp": "^((ERROR|WARNING): .* \\((.*):(\\d+),(\\d+)-\\d+\\))$",
+                    "message": 1,
+                    "file": 3,
+                    "line": 4,
+                    "col": 5,
+                    "severity": 2
+                }
+            ]
+        }
+    ]
+}

.github/problem-matchers/codeql-test-run.json

@@ -0,0 +1,14 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "codeql-test-run",
+            "pattern": [
+                {
+                    "regexp": "(\\[.*\\] FAILED\\((RESULT|COMPILATION)\\) (.*))$",
+                    "file": 3,
+                    "message": 1
+                }
+            ]
+        }
+    ]
+}

.github/problem-matchers/make.json

@@ -0,0 +1,13 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "make",
+            "pattern": [
+                {
+                    "regexp": "^(make: \\*\\*\\* .*)$",
+                    "message": 1
+                }
+            ]
+        }
+    ]
+}

.github/workflows/check-change-note.yml

@@ -10,6 +10,7 @@ on:
       - "*/ql/lib/**/*.qll"
       - "!**/experimental/**"
       - "!ql/**"
+      - "!swift/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:

.github/workflows/check-qldoc.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "*/ql/lib/**"
       - .github/workflows/check-qldoc.yml
+      - .github/actions/fetch-codeql/action.yml
     branches:
       - main
       - "rc/*"
@@ -14,27 +15,23 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Install CodeQL
-        run: |
-          gh extension install github/gh-codeql
-          gh codeql set-channel nightly
-          gh codeql version
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-
       - uses: actions/checkout@v3
         with:
           fetch-depth: 2
 
+      - name: Install CodeQL
+        uses: ./.github/actions/fetch-codeql
+
       - name: Check QLdoc coverage
         shell: bash
         run: |
           EXIT_CODE=0
           # TODO: remove the swift exception from the regex when we fix generated QLdoc
-          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!swift)[a-z]*/ql/lib' || true; } | sort -u)"
+          # TODO: remove the shared exception from the regex when coverage of qlpacks without dbschemes is supported
+          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!(swift|shared))[a-z]*/ql/lib' || true; } | sort -u)"
           for pack_dir in ${changed_lib_packs}; do
             lang="${pack_dir%/ql/lib}"
-            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
+            codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
           done
           git checkout HEAD^
           for pack_dir in ${changed_lib_packs}; do
@@ -42,7 +39,7 @@ jobs:
             # In this case the right thing to do is to skip the check.
             [[ ! -d "${pack_dir}" ]] && continue
             lang="${pack_dir%/ql/lib}"
-            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
+            codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
             awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-current.txt" | sort -u > "${RUNNER_TEMP}/current-undocumented.txt"
             awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-baseline.txt" | sort -u > "${RUNNER_TEMP}/baseline-undocumented.txt"
             UNDOCUMENTED="$(grep -f <(comm -13 "${RUNNER_TEMP}/baseline-undocumented.txt" "${RUNNER_TEMP}/current-undocumented.txt") "${RUNNER_TEMP}/${lang}-current.txt" || true)"

.github/workflows/close-stale.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/stale@v5
+    - uses: actions/stale@v6
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Comment or remove the `Stale` label in order to avoid having this issue closed in 7 days.'

.github/workflows/codeql-analysis.yml

@@ -56,7 +56,7 @@ jobs:
     #    uses a compiled language
 
     - run: |
-       dotnet build csharp /p:UseSharedCompilation=false
+       dotnet build csharp
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@main

.github/workflows/csv-coverage-metrics.yml

@@ -12,6 +12,7 @@ on:
       - main
     paths:
       - ".github/workflows/csv-coverage-metrics.yml"
+      - ".github/actions/fetch-codeql/action.yml"
 
 jobs:
   publish-java:
@@ -54,7 +55,7 @@ jobs:
           DATABASE="${{ runner.temp }}/csharp-database"
           PROJECT="${{ runner.temp }}/csharp-project"
           dotnet new classlib --language=C# --output="$PROJECT"
-          codeql database create "$DATABASE" --language=csharp --source-root="$PROJECT" --command 'dotnet build /t:rebuild csharp-project.csproj /p:UseSharedCompilation=false'
+          codeql database create "$DATABASE" --language=csharp --source-root="$PROJECT" --command 'dotnet build /t:rebuild csharp-project.csproj'
       - name: Capture coverage information
         run: |
           DATABASE="${{ runner.temp }}/csharp-database"

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -3,18 +3,20 @@ name: Check framework coverage changes
 on:
   pull_request:
     paths:
-      - '.github/workflows/csv-coverage-pr-comment.yml'
-      - '*/ql/src/**/*.ql'
-      - '*/ql/src/**/*.qll'
-      - '*/ql/lib/**/*.ql'
-      - '*/ql/lib/**/*.qll'
-      - 'misc/scripts/library-coverage/*.py'
+      - ".github/workflows/csv-coverage-pr-comment.yml"
+      - ".github/workflows/csv-coverage-pr-artifacts.yml"
+      - ".github/actions/fetch-codeql/action.yml"
+      - "*/ql/src/**/*.ql"
+      - "*/ql/src/**/*.qll"
+      - "*/ql/lib/**/*.ql"
+      - "*/ql/lib/**/*.qll"
+      - "misc/scripts/library-coverage/*.py"
       # input data files
-      - '*/documentation/library-coverage/cwe-sink.csv'
-      - '*/documentation/library-coverage/frameworks.csv'
+      - "*/documentation/library-coverage/cwe-sink.csv"
+      - "*/documentation/library-coverage/frameworks.csv"
     branches:
       - main
-      - 'rc/*'
+      - "rc/*"
 
 jobs:
   generate:
@@ -23,77 +25,72 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Dump GitHub context
-      env:
-        GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-      run: echo "$GITHUB_CONTEXT"
-    - name: Clone self (github/codeql) - MERGE
-      uses: actions/checkout@v3
-      with:
-        path: merge
-    - name: Clone self (github/codeql) - BASE
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 2
-        path: base
-    - run: |
-        git checkout HEAD^1
-        git log -1 --format='%H'
-      working-directory: base
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Generate CSV files on merge commit of the PR
-      run: |
-        echo "Running generator on merge"
-        PATH="$PATH:codeql-cli/codeql" python merge/misc/scripts/library-coverage/generate-report.py ci merge merge
-        mkdir out_merge
-        cp framework-coverage-*.csv out_merge/
-        cp framework-coverage-*.rst out_merge/
-    - name: Generate CSV files on base commit of the PR
-      run: |
-        echo "Running generator on base"
-        PATH="$PATH:codeql-cli/codeql" python base/misc/scripts/library-coverage/generate-report.py ci base base
-        mkdir out_base
-        cp framework-coverage-*.csv out_base/
-        cp framework-coverage-*.rst out_base/
-    - name: Generate diff of coverage reports
-      run: |
-        python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
-    - name: Upload CSV package list
-      uses: actions/upload-artifact@v3
-      with:
-        name: csv-framework-coverage-merge
-        path: |
-          out_merge/framework-coverage-*.csv
-          out_merge/framework-coverage-*.rst
-    - name: Upload CSV package list
-      uses: actions/upload-artifact@v3
-      with:
-        name: csv-framework-coverage-base
-        path: |
-          out_base/framework-coverage-*.csv
-          out_base/framework-coverage-*.rst
-    - name: Upload comparison results
-      uses: actions/upload-artifact@v3
-      with:
-        name: comparison
-        path: |
-          comparison.md
-    - name: Save PR number
-      run: |
-        mkdir -p pr
-        echo ${{ github.event.pull_request.number }} > pr/NR
-    - name: Upload PR number
-      uses: actions/upload-artifact@v3
-      with:
-        name: pr
-        path: pr/
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJSON(github.event) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Clone self (github/codeql) - MERGE
+        uses: actions/checkout@v3
+        with:
+          path: merge
+      - name: Clone self (github/codeql) - BASE
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+          path: base
+      - run: |
+          git checkout HEAD^1
+          git log -1 --format='%H'
+        working-directory: base
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Download CodeQL CLI
+        uses: ./merge/.github/actions/fetch-codeql
+      - name: Generate CSV files on merge commit of the PR
+        run: |
+          echo "Running generator on merge"
+          python merge/misc/scripts/library-coverage/generate-report.py ci merge merge
+          mkdir out_merge
+          cp framework-coverage-*.csv out_merge/
+          cp framework-coverage-*.rst out_merge/
+      - name: Generate CSV files on base commit of the PR
+        run: |
+          echo "Running generator on base"
+          python base/misc/scripts/library-coverage/generate-report.py ci base base
+          mkdir out_base
+          cp framework-coverage-*.csv out_base/
+          cp framework-coverage-*.rst out_base/
+      - name: Generate diff of coverage reports
+        run: |
+          python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
+      - name: Upload CSV package list
+        uses: actions/upload-artifact@v3
+        with:
+          name: csv-framework-coverage-merge
+          path: |
+            out_merge/framework-coverage-*.csv
+            out_merge/framework-coverage-*.rst
+      - name: Upload CSV package list
+        uses: actions/upload-artifact@v3
+        with:
+          name: csv-framework-coverage-base
+          path: |
+            out_base/framework-coverage-*.csv
+            out_base/framework-coverage-*.rst
+      - name: Upload comparison results
+        uses: actions/upload-artifact@v3
+        with:
+          name: comparison
+          path: |
+            comparison.md
+      - name: Save PR number
+        run: |
+          mkdir -p pr
+          echo ${{ github.event.pull_request.number }} > pr/NR
+      - name: Upload PR number
+        uses: actions/upload-artifact@v3
+        with:
+          name: pr
+          path: pr/

.github/workflows/csv-coverage-pr-comment.yml

@@ -22,7 +22,7 @@ jobs:
     - name: Clone self (github/codeql)
       uses: actions/checkout@v3
     - name: Set up Python 3.8
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: 3.8
 

.github/workflows/csv-coverage-timeseries.yml

@@ -5,38 +5,29 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
-      with:
-        path: script
-    - name: Clone self (github/codeql) for analysis
-      uses: actions/checkout@v3
-      with:
-        path: codeqlModels
-        fetch-depth: 0
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Build modeled package list
-      run: |
-        CLI=$(realpath "codeql-cli/codeql")
-        echo $CLI
-        PATH="$PATH:$CLI" python script/misc/scripts/library-coverage/generate-timeseries.py codeqlModels
-    - name: Upload timeseries CSV
-      uses: actions/upload-artifact@v3
-      with:
-        name: framework-coverage-timeseries
-        path: framework-coverage-timeseries-*.csv
-
+      - name: Clone self (github/codeql)
+        uses: actions/checkout@v3
+        with:
+          path: script
+      - name: Clone self (github/codeql) for analysis
+        uses: actions/checkout@v3
+        with:
+          path: codeqlModels
+          fetch-depth: 0
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Download CodeQL CLI
+        uses: ./script/.github/actions/fetch-codeql
+      - name: Build modeled package list
+        run: |
+          python script/misc/scripts/library-coverage/generate-timeseries.py codeqlModels
+      - name: Upload timeseries CSV
+        uses: actions/upload-artifact@v3
+        with:
+          name: framework-coverage-timeseries
+          path: framework-coverage-timeseries-*.csv

.github/workflows/csv-coverage-update.yml

@@ -12,33 +12,27 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Dump GitHub context
-      env:
-        GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-      run: echo "$GITHUB_CONTEXT"
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
-      with:
-        path: ql
-        fetch-depth: 0
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJSON(github.event) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Clone self (github/codeql)
+        uses: actions/checkout@v3
+        with:
+          path: ql
+          fetch-depth: 0
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Download CodeQL CLI
+        uses: ./ql/.github/actions/fetch-codeql
+      - name: Generate coverage files
+        run: |
+          python ql/misc/scripts/library-coverage/generate-report.py ci ql ql
 
-    - name: Generate coverage files
-      run: |
-        PATH="$PATH:codeql-cli/codeql" python ql/misc/scripts/library-coverage/generate-report.py ci ql ql
-
-    - name: Create pull request with changes
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-        python ql/misc/scripts/library-coverage/create-pr.py ql "$GITHUB_REPOSITORY"
+      - name: Create pull request with changes
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python ql/misc/scripts/library-coverage/create-pr.py ql "$GITHUB_REPOSITORY"

.github/workflows/csv-coverage.yml

@@ -4,46 +4,39 @@ on:
   workflow_dispatch:
     inputs:
       qlModelShaOverride:
-        description: 'github/codeql repo SHA used for looking up the CSV models'
+        description: "github/codeql repo SHA used for looking up the CSV models"
         required: false
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
-      with:
-        path: script
-    - name: Clone self (github/codeql) for analysis
-      uses: actions/checkout@v3
-      with:
-        path: codeqlModels
-        ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Build modeled package list
-      run: |
-        PATH="$PATH:codeql-cli/codeql" python script/misc/scripts/library-coverage/generate-report.py ci codeqlModels script
-    - name: Upload CSV package list
-      uses: actions/upload-artifact@v3
-      with:
-        name: framework-coverage-csv
-        path: framework-coverage-*.csv
-    - name: Upload RST package list
-      uses: actions/upload-artifact@v3
-      with:
-        name: framework-coverage-rst
-        path: framework-coverage-*.rst
-
+      - name: Clone self (github/codeql)
+        uses: actions/checkout@v3
+        with:
+          path: script
+      - name: Clone self (github/codeql) for analysis
+        uses: actions/checkout@v3
+        with:
+          path: codeqlModels
+          ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Download CodeQL CLI
+        uses: ./script/.github/actions/fetch-codeql
+      - name: Build modeled package list
+        run: |
+          python script/misc/scripts/library-coverage/generate-report.py ci codeqlModels script
+      - name: Upload CSV package list
+        uses: actions/upload-artifact@v3
+        with:
+          name: framework-coverage-csv
+          path: framework-coverage-*.csv
+      - name: Upload RST package list
+        uses: actions/upload-artifact@v3
+        with:
+          name: framework-coverage-rst
+          path: framework-coverage-*.rst

.github/workflows/go-tests.yml

@@ -0,0 +1,114 @@
+name: "Go: Run Tests"
+on:
+  pull_request:
+    paths:
+      - "go/**"
+      - .github/workflows/go-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+jobs:
+  test-linux:
+    name: Test Linux (Ubuntu)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Check that all QL and Go code is autoformatted
+        run: |
+          cd go
+          make check-formatting
+
+      - name: Compile qhelp files to markdown
+        run: |
+          cd go
+          env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown
+
+      - name: Upload qhelp markdown
+        uses: actions/upload-artifact@v2
+        with:
+          name: qhelp-markdown
+          path: go/qhelp-out/**/*.md
+
+      - name: Test
+        run: |
+          cd go
+          make test
+
+  test-mac:
+    name: Test MacOS
+    runs-on: macos-latest
+    steps:
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Test
+        run: |
+          cd go
+          make test
+
+  test-win:
+    name: Test Windows
+    runs-on: windows-2019
+    steps:
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Test
+        run: |
+          cd go
+          make test

.github/workflows/js-ml-tests.yml

@@ -5,6 +5,8 @@ on:
     paths:
       - "javascript/ql/experimental/adaptivethreatmodeling/**"
       - .github/workflows/js-ml-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
@@ -12,6 +14,9 @@ on:
     paths:
       - "javascript/ql/experimental/adaptivethreatmodeling/**"
       - .github/workflows/js-ml-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+  workflow_dispatch:
 
 defaults:
   run:

.github/workflows/labeler.yml

@@ -4,6 +4,9 @@ on:
 
 jobs:
   triage:
+    permissions:
+      contents: read
+      pull-requests: write
     runs-on: ubuntu-latest
     steps:
     - uses: actions/labeler@v4

.github/workflows/mad_modelDiff.yml

@@ -61,7 +61,7 @@ jobs:
             DATABASE=$2
             cd codeql-$QL_VARIANT
             SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
+            python java/ql/src/utils/model-generator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $MODELS/${SHORTNAME}.qll
             mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
             cd ..
           }

.github/workflows/mad_regenerate-models.yml

@@ -9,6 +9,7 @@ on:
       - main
     paths:
       - ".github/workflows/mad_regenerate-models.yml"
+      - ".github/actions/fetch-codeql/action.yml"
 
 jobs:
   regenerate-models:
@@ -20,7 +21,7 @@ jobs:
         ref: ["placeholder"]
         include:
           - slug: "apache/commons-io"
-            ref: "8985de8fe74f6622a419b37a6eed0dbc484dc128"
+            ref: "13258ce2d07aa0e764bbaa8020af4dcd3a02a620"
         exclude:
           - slug: "placeholder"
             ref: "placeholder"

.github/workflows/ql-for-ql-build.yml

@@ -5,21 +5,28 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+    paths:
+      - "ql/**"
+      - "**.qll"
+      - "**.ql"
+      - "**.dbscheme"
+      - "**/qlpack.yml"
+      - ".github/workflows/ql-for-ql-build.yml"
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  queries:
-    runs-on: ubuntu-latest
+  analyze:
+    runs-on: ubuntu-latest-xl
     steps:
+      ### Build the queries ###
       - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: javascript # does not matter
-          tools: latest
       - name: Get CodeQL version
         id: get-codeql-version
         run: |
@@ -27,37 +34,37 @@ jobs:
         shell: bash
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+      - name: Cache entire pack
+        id: cache-pack
+        uses: actions/cache@v3
+        with:
+          path: ${{ runner.temp }}/pack
+          key: ${{ runner.os }}-pack-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
       - name: Cache queries
+        if: steps.cache-pack.outputs.cache-hit != 'true'
         id: cache-queries
         uses: actions/cache@v3
         with:
-          path: ${{ runner.temp }}/query-pack.zip
-          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
+          path: ${{ runner.temp }}/queries
+          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
       - name: Build query pack
-        if: steps.cache-queries.outputs.cache-hit != 'true'
+        if: steps.cache-queries.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         run: |
           cd ql/ql/src
-          "${CODEQL}" pack create
-          cd .codeql/pack/codeql/ql/0.0.0
-          zip "${PACKZIP}" -r .
+          "${CODEQL}" pack create -j 16
+          mv .codeql/pack/codeql/ql/0.0.0 ${{ runner.temp }}/queries
+        env:
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+      - name: Move cache queries to pack
+        if: steps.cache-pack.outputs.cache-hit != 'true'
+        run: |
+          cp -r ${{ runner.temp }}/queries ${{ runner.temp }}/pack
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-          PACKZIP: ${{ runner.temp }}/query-pack.zip
-      - name: Upload query pack
-        uses: actions/upload-artifact@v3
-        with:
-          name: query-pack-zip
-          path: ${{ runner.temp }}/query-pack.zip
 
-  extractors:
-    strategy:
-      fail-fast: false
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
+      ### Build the extractor ###
       - name: Cache entire extractor
+        if: steps.cache-pack.outputs.cache-hit != 'true'
         id: cache-extractor
         uses: actions/cache@v3
         with:
@@ -68,7 +75,7 @@ jobs:
             ql/target/release/ql-extractor.exe
           key: ${{ runner.os }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
       - name: Cache cargo
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         uses: actions/cache@v3
         with:
           path: |
@@ -77,127 +84,87 @@ jobs:
             ql/target
           key: ${{ runner.os }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Check formatting
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         run: cd ql; cargo fmt --all -- --check
       - name: Build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         run: cd ql; cargo build --verbose
       - name: Run tests
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         run: cd ql; cargo test --verbose
       - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         run: cd ql; cargo build --release
       - name: Generate dbscheme
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v3
-        with:
-          name: extractor-ubuntu-latest
-          path: |
-            ql/target/release/ql-autobuilder
-            ql/target/release/ql-autobuilder.exe
-            ql/target/release/ql-extractor
-            ql/target/release/ql-extractor.exe
-          retention-days: 1
-  package:
-    runs-on: ubuntu-latest
 
-    needs:
-      - extractors
-      - queries
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
-        with:
-          name: query-pack-zip
-          path: query-pack-zip
-      - uses: actions/download-artifact@v3
-        with:
-          name: extractor-ubuntu-latest
-          path: linux64
-      - run: |
-          unzip query-pack-zip/*.zip -d pack
-          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats pack/
-          mkdir -p pack/tools/linux64
-          if [[ -f linux64/ql-autobuilder ]]; then
-            cp linux64/ql-autobuilder pack/tools/linux64/autobuilder
-            chmod +x pack/tools/linux64/autobuilder
-          fi
-          if [[ -f linux64/ql-extractor ]]; then
-            cp linux64/ql-extractor pack/tools/linux64/extractor
-            chmod +x pack/tools/linux64/extractor
-          fi
-          cd pack
-          zip -rq ../codeql-ql.zip .
-      - uses: actions/upload-artifact@v3
-        with:
-          name: codeql-ql-pack
-          path: codeql-ql.zip
-          retention-days: 1
-  analyze:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        folder: [cpp, csharp, java, javascript, python, ql, ruby, swift]
-
-    needs:
-      - package
-
-    steps:
-      - name: Download pack
-        uses: actions/download-artifact@v3
-        with:
-          name: codeql-ql-pack
-          path: ${{ runner.temp }}/codeql-ql-pack-artifact
-
-      - name: Prepare pack
+      ### Package the queries and extractor ###
+      - name: Package pack
+        if: steps.cache-pack.outputs.cache-hit != 'true'
         run: |
-          unzip "${PACK_ARTIFACT}/*.zip" -d "${PACK}"
+          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats ${PACK}/
+          mkdir -p ${PACK}/tools/linux64
+          cp ql/target/release/ql-autobuilder  ${PACK}/tools/linux64/autobuilder
+          cp ql/target/release/ql-extractor ${PACK}/tools/linux64/extractor
+          chmod +x ${PACK}/tools/linux64/autobuilder
+          chmod +x ${PACK}/tools/linux64/extractor
         env:
-          PACK_ARTIFACT: ${{ runner.temp }}/codeql-ql-pack-artifact
           PACK: ${{ runner.temp }}/pack
+
+      ### Run the analysis ###
       - name: Hack codeql-action options
         run: |
-          JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
+          JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .resolve.languages=["--search-path", $pack] | .database.init=["--search-path", $pack]')
           echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
         env:
           PACK: ${{ runner.temp }}/pack
 
-      - name: Checkout repository
-        uses: actions/checkout@v3
       - name: Create CodeQL config file
         run: |
-          echo "paths:" > ${CONF}
-          echo "  - ${FOLDER}" >> ${CONF}
           echo "paths-ignore:" >> ${CONF}
-          echo "  - ql/ql/test" >> ${CONF} 
+          echo "  - ql/ql/test" >> ${CONF}
+          echo "  - \"*/ql/lib/upgrades/\"" >> ${CONF}
           echo "disable-default-queries: true" >> ${CONF}
-          echo "packs:" >> ${CONF}
-          echo "  - codeql/ql" >> ${CONF}
+          echo "queries:" >> ${CONF}
+          echo "  - uses: ./ql/ql/src/codeql-suites/ql-code-scanning.qls" >> ${CONF}
           echo "Config file: "
           cat ${CONF}
-        env: 
+        env:
           CONF: ./ql-for-ql-config.yml
-          FOLDER: ${{ matrix.folder }}
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: ql
           db-location: ${{ runner.temp }}/db
           config-file: ./ql-for-ql-config.yml
-          tools: latest
+      - name: Move pack cache
+        run: |
+          cp -r ${PACK}/.cache ql/ql/src/.cache
+        env:
+          PACK: ${{ runner.temp }}/pack
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@aa93aea877e5fb8841bcb1193f672abf6e9f2980
-        with: 
-          category: "ql-for-ql-${{ matrix.folder }}"
+        uses: github/codeql-action/analyze@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        with:
+          category: "ql-for-ql"
       - name: Copy sarif file to CWD
-        run: cp ../results/ql.sarif ./${{ matrix.folder }}.sarif
+        run: cp ../results/ql.sarif ./ql-for-ql.sarif
+      - name: Fixup the $scema in sarif  # Until https://github.com/microsoft/sarif-vscode-extension/pull/436/ is part in a stable release
+        run: |
+          sed -i 's/\$schema.*/\$schema": "https:\/\/raw.githubusercontent.com\/oasis-tcs\/sarif-spec\/master\/Schemata\/sarif-schema-2.1.0",/' ql-for-ql.sarif
       - name: Sarif as artifact
         uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.folder }}.sarif
-          path: ${{ matrix.folder }}.sarif
-
+          name: ql-for-ql.sarif
+          path: ql-for-ql.sarif
+      - name: Split out the sarif file into langs
+        run: |
+          mkdir split-sarif
+          node ./ql/scripts/split-sarif.js ql-for-ql.sarif split-sarif
+      - name: Upload langs as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ql-for-ql-langs
+          path: split-sarif
+          retention-days: 1

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -19,14 +19,13 @@ jobs:
       matrix:
         repo:
           - github/codeql
-          - github/codeql-go
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v3
@@ -37,7 +36,7 @@ jobs:
             ql/target
           key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Build Extractor
-        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
+        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./scripts/create-extractor-pack.sh
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Checkout ${{ matrix.repo }}
@@ -77,7 +76,7 @@ jobs:
           path: stats
       - run: |
           python -m pip install --user lxml
-          find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ql/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
+          find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ruby/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
       - uses: actions/upload-artifact@v3
         with:
           name: ql.dbscheme.stats

.github/workflows/ql-for-ql-tests.yml

@@ -5,10 +5,12 @@ on:
     branches: [main]
     paths:
       - "ql/**"
+      - codeql-workspace.yml
   pull_request:
     branches: [main]
     paths:
       - "ql/**"
+      - codeql-workspace.yml
 
 env:
   CARGO_TERM_COLOR: always
@@ -20,7 +22,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v3
@@ -34,7 +36,7 @@ jobs:
         run: |
           cd ql;
           codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
-          env "PATH=$PATH:$codeqlpath" ./create-extractor-pack.sh
+          env "PATH=$PATH:$codeqlpath" ./scripts/create-extractor-pack.sh
       - name: Run QL tests
         run: |
           "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test
@@ -42,7 +44,7 @@ jobs:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Check QL formatting
         run: |
-          find ql/ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
+          find ql/ql/src "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Check QL compilation

.github/workflows/query-list.yml

@@ -5,9 +5,12 @@ on:
     branches:
      - main
      - 'rc/**'
+    tags:
+     - 'codeql-cli/*'
   pull_request:
     paths:
       - '.github/workflows/query-list.yml'
+      - '.github/actions/fetch-codeql/action.yml'
       - 'misc/scripts/generate-code-scanning-query-list.py'
 
 jobs:
@@ -20,20 +23,13 @@ jobs:
       uses: actions/checkout@v3
       with:
         path: codeql 
-    - name: Clone github/codeql-go
-      uses: actions/checkout@v3
-      with:
-        repository: 'github/codeql-go'
-        path: codeql-go
     - name: Set up Python 3.8
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: 3.8
     - name: Download CodeQL CLI
       # Look under the `codeql` directory, as this is where we checked out the `github/codeql` repo
       uses: ./codeql/.github/actions/fetch-codeql
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
     - name: Build code scanning query list
       run: |
         python codeql/misc/scripts/generate-code-scanning-query-list.py > code-scanning-query-list.csv

.github/workflows/ruby-build.yml

@@ -5,6 +5,8 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-build.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
@@ -12,6 +14,8 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-build.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
@@ -88,19 +92,15 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Fetch CodeQL
-        run: |
-          LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
-          gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
-          unzip -q codeql-linux64.zip
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
+        uses: ./.github/actions/fetch-codeql
       - name: Build Query Pack
         run: |
-          codeql/codeql pack create ql/lib --output target/packs
-          codeql/codeql pack install ql/src
-          codeql/codeql pack create ql/src --output target/packs
+          codeql pack create ../shared/ssa --output target/packs
+          codeql pack create ql/lib --output target/packs
+          codeql pack install ql/src
+          codeql pack create ql/src --output target/packs
           PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
-          codeql/codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
+          codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
           (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
       - uses: actions/upload-artifact@v3
         with:
@@ -177,19 +177,15 @@ jobs:
     runs-on: ${{ matrix.os }}
     needs: [package]
     steps:
+      - uses: actions/checkout@v3
+      - name: Fetch CodeQL
+        uses: ./.github/actions/fetch-codeql
+
       - uses: actions/checkout@v3
         with:
           repository: Shopify/example-ruby-app
           ref: 67a0decc5eb550f3a9228eda53925c3afd40dfe9
-      - name: Fetch CodeQL
-        shell: bash
-        run: |
-          LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
-          gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql.zip "$LATEST"
-          unzip -q codeql.zip
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        working-directory: ${{ runner.temp }}
+
       - name: Download Ruby bundle
         uses: actions/download-artifact@v3
         with:
@@ -201,7 +197,7 @@ jobs:
       - name: Prepare test files
         shell: bash
         run: |
-          echo "import ruby select count(File f)" > "test.ql"
+          echo "import codeql.ruby.AST select count(File f)" > "test.ql"
           echo "| 4 |" > "test.expected"
           echo 'name: sample-tests
           version: 0.0.0
@@ -213,12 +209,12 @@ jobs:
       - name: Run QL test
         shell: bash
         run: |
-          "${{ runner.temp }}/codeql/codeql" test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
+          codeql test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
       - name: Create database
         shell: bash
         run: |
-          "${{ runner.temp }}/codeql/codeql" database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
+          codeql database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
       - name: Analyze database
         shell: bash
         run: |
-          "${{ runner.temp }}/codeql/codeql" database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls
+          codeql database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/ruby-qltest.yml

@@ -5,6 +5,8 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-qltest.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
@@ -12,6 +14,8 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-qltest.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"

.github/workflows/swift-codegen.yml

@@ -4,7 +4,10 @@ on:
   pull_request:
     paths:
       - "swift/**"
+      - "misc/bazel/**"
+      - "*.bazel*"
       - .github/workflows/swift-codegen.yml
+      - .github/actions/fetch-codeql/action.yml
     branches:
       - main
 
@@ -15,11 +18,22 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v3
+      - uses: pre-commit/action@v3.0.0
+        name: Check that python code is properly formatted
+        with:
+          extra_args: autopep8 --all-files
       - name: Run unit tests
         run: |
           bazel test //swift/codegen/test --test_output=errors
-      - name: Check that code was generated
+      - uses: pre-commit/action@v3.0.0
+        name: Check that QL generated code was checked in
+        with:
+          extra_args: swift-codegen --all-files
+      - name: Generate C++ files
         run: |
-          bazel run //swift/codegen
-          git add swift
-          git diff --exit-code --stat HEAD
+          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-cpp-files
+      - uses: actions/upload-artifact@v3
+        with:
+          name: swift-generated-cpp-files
+          path: swift-generated-cpp-files/**

.github/workflows/swift-integration-tests.yml

@@ -0,0 +1,45 @@
+name: "Swift: Run Integration Tests"
+
+on:
+  pull_request:
+    paths:
+      - "swift/**"
+      - "misc/bazel/**"
+      - "*.bazel*"
+      - .github/workflows/swift-integration-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+defaults:
+  run:
+    working-directory: swift
+
+jobs:
+  integration-tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-20.04
+#          - macos-latest  TODO
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v3
+      - name: Build Swift extractor
+        run: |
+          bazel run //swift:create-extractor-pack
+      - name: Get Swift version
+        id: get_swift_version
+        run: |
+          VERSION=$(bazel run //swift/extractor -- --version | sed -ne 's/.*version \(\S*\).*/\1/p')
+          echo "::set-output name=version::$VERSION"
+      - uses: swift-actions/setup-swift@v1
+        with:
+          swift-version: "${{steps.get_swift_version.outputs.version}}"
+      - name: Run integration tests
+        run: |
+          python integration-tests/runner.py

.github/workflows/swift-qltest.yml

@@ -4,7 +4,11 @@ on:
   pull_request:
     paths:
       - "swift/**"
+      - "misc/bazel/**"
+      - "*.bazel*"
       - .github/workflows/swift-qltest.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
     branches:
       - main
 defaults:

.github/workflows/validate-change-notes.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "*/ql/*/change-notes/**/*"
       - ".github/workflows/validate-change-notes.yml"
+      - ".github/actions/fetch-codeql/action.yml"
     branches:
       - main
       - "rc/*"
@@ -12,6 +13,7 @@ on:
     paths:
       - "*/ql/*/change-notes/**/*"
       - ".github/workflows/validate-change-notes.yml"
+      - ".github/actions/fetch-codeql/action.yml"
 
 jobs:
   check-change-note:

.gitignore

@@ -9,6 +9,7 @@
 # qltest projects and artifacts
 */ql/test/**/*.testproj
 */ql/test/**/*.actual
+*/ql/test/**/go.sum
 
 # Visual studio temporaries, except a file used by QL4VS
 .vs/*
@@ -42,3 +43,21 @@ csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
 
 # CLion project files
 /.clwb
+
+# Go build artifacts
+go/build/*
+
+# Go binaries
+go/tools/bin
+go/tools/linux64
+go/tools/osx64
+go/tools/win64
+go/tools/tokenizer.jar
+go/main
+
+# node_modules folders except in the JS test suite
+node_modules/
+!/javascript/ql/test/**/node_modules/
+
+# Temporary folders for working with generated models
+.model-temp

.lgtm.yml

@@ -6,6 +6,7 @@ path_classifiers:
   test:
   - csharp/ql/src
   - csharp/ql/test
+  - go/ql/test
   - javascript/extractor/parser-tests
   - javascript/extractor/tests
   - javascript/ql/src
@@ -13,6 +14,9 @@ path_classifiers:
   - python/ql/src
   - python/ql/test
 
+  example:
+  - go/ql/src
+
 queries:
   - include: "*"
 

.pre-commit-config.yaml

@@ -15,6 +15,12 @@ repos:
       - id: clang-format
         files: ^swift/.*\.(h|c|cpp)$
 
+  - repo: https://github.com/pre-commit/mirrors-autopep8
+    rev: v1.6.0
+    hooks:
+      - id: autopep8
+        files: ^swift/codegen/.*\.py
+
   - repo: local
     hooks:
       - id: codeql-format
@@ -25,7 +31,7 @@ repos:
 
       - id: sync-files
         name: Fix files required to be identical
-        files: \.(qll?|qhelp)$
+        files: \.(qll?|qhelp|swift)$
         language: system
         entry: python3 config/sync-files.py --latest
         pass_filenames: false
@@ -40,7 +46,7 @@ repos:
         name: Run Swift checked in code generation
         files: ^swift/(codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements))
         language: system
-        entry: bazel run //swift/codegen
+        entry: bazel run //swift/codegen -- --quiet
         pass_filenames: false
 
       - id: swift-codegen-unit-tests

CODEOWNERS

@@ -1,5 +1,6 @@
 /cpp/ @github/codeql-c-analysis
 /csharp/ @github/codeql-csharp
+/go/ @github/codeql-go
 /java/ @github/codeql-java
 /javascript/ @github/codeql-javascript
 /python/ @github/codeql-python
@@ -27,8 +28,10 @@
 # QL for QL reviewers
 /ql/ @github/codeql-ql-for-ql-reviewers
 
-# Bazel
-**/*.bazel @github/codeql-ci-reviewers
+# Bazel (excluding BUILD.bazel files)
+WORKSPACE.bazel @github/codeql-ci-reviewers
+.bazelversion @github/codeql-ci-reviewers
+.bazelrc @github/codeql-ci-reviewers
 **/*.bzl @github/codeql-ci-reviewers
 
 # Documentation etc
@@ -37,6 +40,8 @@
 
 # Workflows
 /.github/workflows/ @github/codeql-ci-reviewers
+/.github/workflows/go-* @github/codeql-go
 /.github/workflows/js-ml-tests.yml @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
 /.github/workflows/ruby-* @github/codeql-ruby
+/.github/workflows/swift-* @github/codeql-c

README.md

@@ -1,6 +1,6 @@
 # CodeQL
 
-This open source repository contains the standard CodeQL libraries and queries that power [GitHub Advanced Security](https://github.com/features/security/code) and the other application security products that [GitHub](https://github.com/features/security/) makes available to its customers worldwide. For the queries, libraries, and extractor that power Go analysis, visit the [CodeQL for Go repository](https://github.com/github/codeql-go).
+This open source repository contains the standard CodeQL libraries and queries that power [GitHub Advanced Security](https://github.com/features/security/code) and the other application security products that [GitHub](https://github.com/features/security/) makes available to its customers worldwide.
 
 ## How do I learn CodeQL and run queries?
 

codeql-workspace.yml

@@ -0,0 +1,33 @@
+provide:
+  - "*/ql/src/qlpack.yml"
+  - "*/ql/lib/qlpack.yml"
+  - "*/ql/test/qlpack.yml"
+  - "*/ql/examples/qlpack.yml"
+  - "*/ql/consistency-queries/qlpack.yml"
+  - "shared/*/qlpack.yml"
+  - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
+  - "go/ql/config/legacy-support/qlpack.yml"
+  - "go/build/codeql-extractor-go/codeql-extractor.yml"
+  - "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml"
+  # This pack is explicitly excluded from the workspace since most users
+  # will want to use a version of this pack from the package cache. Internal
+  # users can uncomment the following line and place a custom ML model
+  # in the corresponding pack to test a custom ML model within their local
+  # checkout.
+  # - "javascript/ql/experimental/adaptivethreatmodeling/model/qlpack.yml"
+  - "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml"
+  - "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml"
+  - "csharp/ql/campaigns/Solorigate/lib/qlpack.yml"
+  - "csharp/ql/campaigns/Solorigate/src/qlpack.yml"
+  - "csharp/ql/campaigns/Solorigate/test/qlpack.yml"
+  - "misc/legacy-support/*/qlpack.yml"
+  - "misc/suite-helpers/qlpack.yml"
+  - "ruby/extractor-pack/codeql-extractor.yml"
+  - "swift/extractor-pack/codeql-extractor.yml"
+  - "ql/extractor-pack/codeql-extractor.ym"
+
+versionPolicies:
+  default:
+    requireChangeNotes: true
+    committedPrereleaseSuffix: dev
+    committedVersion: nextPatchRelease

config/identical-files.json

@@ -17,26 +17,35 @@
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl3.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl4.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl2.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplForContentDataFlow.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForLibraries.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForLibraries.qll",
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForHttpClientLibraries.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"
   ],
   "DataFlow Java/C++/C#/Python Common": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll"
   ],
   "TaintTracking::Configuration Java/C++/C#/Python": [
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
@@ -44,6 +53,9 @@
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
@@ -57,20 +69,25 @@
     "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking3/TaintTrackingImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking4/TaintTrackingImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttrackingforlibraries/TaintTrackingImpl.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttrackingforlibraries/TaintTrackingImpl.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
   ],
   "DataFlow Java/C++/C#/Python Consistency checks": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplConsistency.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll",
+    "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplConsistency.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplConsistency.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplConsistency.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplConsistency.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplConsistency.qll"
   ],
-  "DataFlow Java/C# Flow Summaries": [
+  "DataFlow Java/C#/Ruby/Python/Swift Flow Summaries": [
     "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/FlowSummaryImpl.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/FlowSummaryImpl.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/FlowSummaryImpl.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/FlowSummaryImpl.qll"
   ],
   "SsaReadPosition Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll",
@@ -385,7 +402,9 @@
     "java/ql/test/TestUtilities/InlineExpectationsTest.qll",
     "python/ql/test/TestUtilities/InlineExpectationsTest.qll",
     "ruby/ql/test/TestUtilities/InlineExpectationsTest.qll",
-    "ql/ql/test/TestUtilities/InlineExpectationsTest.qll"
+    "ql/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "go/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "swift/ql/test/TestUtilities/InlineExpectationsTest.qll"
   ],
   "C++ ExternalAPIs": [
     "cpp/ql/src/Security/CWE/CWE-020/ExternalAPIs.qll",
@@ -446,19 +465,11 @@
     "python/ql/src/Lexical/CommentedOutCodeReferences.inc.qhelp"
   ],
   "IDE Contextual Queries": [
-    "cpp/ql/src/IDEContextual.qll",
-    "csharp/ql/src/IDEContextual.qll",
-    "java/ql/src/IDEContextual.qll",
-    "javascript/ql/src/IDEContextual.qll",
-    "python/ql/src/analysis/IDEContextual.qll"
-  ],
-  "SSA C#": [
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/SsaImplCommon.qll",
-    "csharp/ql/lib/semmle/code/csharp/controlflow/internal/pressa/SsaImplCommon.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/basessa/SsaImplCommon.qll",
-    "csharp/ql/lib/semmle/code/cil/internal/SsaImplCommon.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/SsaImplCommon.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaImplCommon.qll"
+    "cpp/ql/lib/IDEContextual.qll",
+    "csharp/ql/lib/IDEContextual.qll",
+    "java/ql/lib/IDEContextual.qll",
+    "javascript/ql/lib/IDEContextual.qll",
+    "python/ql/lib/analysis/IDEContextual.qll"
   ],
   "CryptoAlgorithms Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/CryptoAlgorithms.qll",
@@ -475,29 +486,44 @@
     "python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll",
     "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll"
   ],
-  "ReDoS Util Python/JS/Ruby": [
-    "javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll",
-    "python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll",
-    "ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll"
+  "ReDoS Util Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/NfaUtils.qll",
+    "python/ql/lib/semmle/python/security/regexp/NfaUtils.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/NfaUtils.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/NfaUtils.qll"
   ],
-  "ReDoS Exponential Python/JS/Ruby": [
-    "javascript/ql/lib/semmle/javascript/security/performance/ExponentialBackTracking.qll",
-    "python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll",
-    "ruby/ql/lib/codeql/ruby/security/performance/ExponentialBackTracking.qll"
+  "ReDoS Exponential Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/ExponentialBackTracking.qll",
+    "python/ql/lib/semmle/python/security/regexp/ExponentialBackTracking.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/ExponentialBackTracking.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/ExponentialBackTracking.qll"
   ],
-  "ReDoS Polynomial Python/JS/Ruby": [
-    "javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll",
-    "python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll",
-    "ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll"
+  "ReDoS Polynomial Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/SuperlinearBackTracking.qll",
+    "python/ql/lib/semmle/python/security/regexp/SuperlinearBackTracking.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/SuperlinearBackTracking.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/SuperlinearBackTracking.qll"
+  ],
+  "RegexpMatching Python/JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/RegexpMatching.qll",
+    "python/ql/lib/semmle/python/security/regexp/RegexpMatching.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/RegexpMatching.qll"
   ],
   "BadTagFilterQuery Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll",
     "python/ql/lib/semmle/python/security/BadTagFilterQuery.qll",
     "ruby/ql/lib/codeql/ruby/security/BadTagFilterQuery.qll"
   ],
+  "OverlyLargeRange Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/OverlyLargeRangeQuery.qll",
+    "python/ql/lib/semmle/python/security/OverlyLargeRangeQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/OverlyLargeRangeQuery.qll",
+    "java/ql/lib/semmle/code/java/security/OverlyLargeRangeQuery.qll"
+  ],
   "CFG": [
     "csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll",
-    "ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll"
+    "ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll",
+    "swift/ql/lib/codeql/swift/controlflow/internal/ControlFlowGraphImplShared.qll"
   ],
   "TypeTracker": [
     "python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll",
@@ -515,7 +541,9 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/AccessPathSyntax.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/AccessPathSyntax.qll",
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/AccessPathSyntax.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/AccessPathSyntax.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/AccessPathSyntax.qll"
   ],
   "IncompleteUrlSubstringSanitization": [
     "javascript/ql/src/Security/CWE-020/IncompleteUrlSubstringSanitization.qll",
@@ -533,7 +561,8 @@
   ],
   "ApiGraphModels": [
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll",
-    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll"
+    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll",
+    "python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll"
   ],
   "TaintedFormatStringQuery Ruby/JS": [
     "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringQuery.qll",
@@ -550,5 +579,29 @@
   "HttpToFileAccessCustomizations JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/dataflow/HttpToFileAccessCustomizations.qll",
     "ruby/ql/lib/codeql/ruby/security/HttpToFileAccessCustomizations.qll"
+  ],
+  "Typo database": [
+    "javascript/ql/src/Expressions/TypoDatabase.qll",
+    "ql/ql/src/codeql_ql/style/TypoDatabase.qll"
+  ],
+  "Swift declarations test file": [
+    "swift/ql/test/extractor-tests/declarations/declarations.swift",
+    "swift/ql/test/library-tests/ast/declarations.swift"
+  ],
+  "Swift statements test file": [
+    "swift/ql/test/extractor-tests/statements/statements.swift",
+    "swift/ql/test/library-tests/ast/statements.swift"
+  ],
+  "Swift expressions test file": [
+    "swift/ql/test/extractor-tests/expressions/expressions.swift",
+    "swift/ql/test/library-tests/ast/expressions.swift"
+  ],
+  "Swift patterns test file": [
+    "swift/ql/test/extractor-tests/patterns/patterns.swift",
+    "swift/ql/test/library-tests/ast/patterns.swift"
+  ],
+  "IncompleteMultiCharacterSanitization JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/IncompleteMultiCharacterSanitizationQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/IncompleteMultiCharacterSanitizationQuery.qll"
   ]
-}
+}

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -299,7 +299,7 @@ namespace Semmle.Autobuild.Cpp.Tests
         {
             Actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test.sln -DisableParallelProcessing"] = 1;
             Actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test.sln -DisableParallelProcessing"] = 0;
-            Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /p:UseSharedCompilation=false /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0;
+            Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0;
             Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0;

cpp/downgrades/19e31bf071f588bb7efd1e4d5a185ce4f6fbbd84/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add relation for tracking C++ braced initializers
+compatibility: full
+braced_initialisers.rel: delete

cpp/downgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/exprs.ql

@@ -0,0 +1,17 @@
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Location extends @location_expr {
+  string toString() { none() }
+}
+
+predicate isExprWithNewBuiltin(Expr expr) {
+  exists(int kind | exprs(expr, kind, _) | 330 <= kind and kind <= 334)
+}
+
+from Expr expr, int kind, int kind_new, Location location
+where
+  exprs(expr, kind, location) and
+  if isExprWithNewBuiltin(expr) then kind_new = 0 else kind_new = kind
+select expr, kind_new, location

cpp/downgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add new builtin operations
+compatibility: partial
+exprs.rel: run exprs.qlo

cpp/downgrades/34549c3b0937002f11037d01822ebe99442c1402/attribute_args.ql

@@ -0,0 +1,17 @@
+class AttributeArgument extends @attribute_arg {
+  string toString() { none() }
+}
+
+class Attribute extends @attribute {
+  string toString() { none() }
+}
+
+class LocationDefault extends @location_default {
+  string toString() { none() }
+}
+
+from AttributeArgument arg, int kind, Attribute attr, int index, LocationDefault location
+where
+  attribute_args(arg, kind, attr, index, location) and
+  not arg instanceof @attribute_arg_constant_expr
+select arg, kind, attr, index, location

cpp/downgrades/34549c3b0937002f11037d01822ebe99442c1402/upgrade.properties

@@ -0,0 +1,4 @@
+description: Support all constant attribute arguments
+compatibility: backwards
+attribute_arg_constant.rel: delete
+attribute_args.rel: run attribute_args.qlo

cpp/downgrades/625f706f2a44ae8dc3fc168bfe2637e65c30b012/exprs.ql

@@ -0,0 +1,17 @@
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Location extends @location_expr {
+  string toString() { none() }
+}
+
+predicate isExprWithNewBuiltin(Expr expr) {
+  exists(int kind | exprs(expr, kind, _) | 336 <= kind and kind <= 362)
+}
+
+from Expr expr, int kind, int kind_new, Location location
+where
+  exprs(expr, kind, location) and
+  if isExprWithNewBuiltin(expr) then kind_new = 1 else kind_new = kind
+select expr, kind_new, location

cpp/downgrades/625f706f2a44ae8dc3fc168bfe2637e65c30b012/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add new builtin operations
+compatibility: partial
+exprs.rel: run exprs.qlo

cpp/downgrades/73af5058c6899dcdb05754c27ca966aeb3a68c94/exprs.ql

@@ -0,0 +1,13 @@
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Location extends @location_expr {
+  string toString() { none() }
+}
+
+from Expr expr, int kind, int kind_new, Location location
+where
+  exprs(expr, kind, location) and
+  if expr instanceof @blockassignexpr then kind_new = 0 else kind_new = kind
+select expr, kind_new, location

cpp/downgrades/73af5058c6899dcdb05754c27ca966aeb3a68c94/upgrade.properties

@@ -0,0 +1,3 @@
+description: Support block assignment
+compatibility: partial
+exprs.rel: run exprs.qlo

cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/exprparents.ql

@@ -0,0 +1,21 @@
+class Element extends @element {
+  string toString() { none() }
+}
+
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Stmt extends @stmt {
+  string toString() { none() }
+}
+
+predicate isStmtWithInitializer(Stmt stmt) {
+  exists(int kind | stmts(stmt, kind, _) | kind = 2 or kind = 11 or kind = 35)
+}
+
+from Expr child, int index, int index_new, Element parent
+where
+  exprparents(child, index, parent) and
+  if isStmtWithInitializer(parent) then index_new = index - 1 else index_new = index
+select child, index_new, parent

cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/stmtparents.ql

@@ -0,0 +1,22 @@
+class Element extends @element {
+  string toString() { none() }
+}
+
+class Stmt extends @stmt {
+  string toString() { none() }
+}
+
+predicate isStmtWithInitializer(Stmt stmt) {
+  exists(int kind | stmts(stmt, kind, _) | kind = 2 or kind = 11 or kind = 35)
+}
+
+from Stmt child, int index, int index_new, Element parent
+where
+  stmtparents(child, index, parent) and
+  (
+    not isStmtWithInitializer(parent)
+    or
+    index > 0
+  ) and
+  if isStmtWithInitializer(parent) then index_new = index - 1 else index_new = index
+select child, index_new, parent

cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/upgrade.properties

@@ -0,0 +1,6 @@
+description: Support C++17 if and switch initializers
+compatibility: partial
+if_initialization.rel: delete
+switch_initialization.rel: delete
+exprparents.rel: run exprparents.qlo
+stmtparents.rel: run stmtparents.qlo

cpp/downgrades/f96ad9b2da43bbc9e55a72a165febd270ae07981/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add relation for orphaned local variables
+compatibility: full
+orphaned_variables.rel: delete

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,86 @@
+## 0.4.0
+
+### Deprecated APIs
+
+* Some classes/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.
+
+### New Features
+
+* Added subclasses of `BuiltInOperations` for `__is_same`, `__is_function`, `__is_layout_compatible`, `__is_pointer_interconvertible_base_of`, `__is_array`, `__array_rank`, `__array_extent`, `__is_arithmetic`, `__is_complete_type`, `__is_compound`, `__is_const`, `__is_floating_point`, `__is_fundamental`, `__is_integral`, `__is_lvalue_reference`, `__is_member_function_pointer`, `__is_member_object_pointer`, `__is_member_pointer`, `__is_object`, `__is_pointer`, `__is_reference`, `__is_rvalue_reference`, `__is_scalar`, `__is_signed`, `__is_unsigned`, `__is_void`, and `__is_volatile`.
+
+### Bug Fixes
+
+* Fixed an issue in the taint tracking analysis where implicit reads were not allowed by default in sinks or additional taint steps that used flow states.
+
+## 0.3.5
+
+## 0.3.4
+
+### Deprecated APIs
+
+* Many classes/predicates/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.
+
+### New Features
+
+* Added support for getting the link targets of global and namespace variables.
+* Added a `BlockAssignExpr` class, which models a `memcpy`-like operation used in compiler generated copy/move constructors and assignment operations.
+
+### Minor Analysis Improvements
+
+* All deprecated predicates/classes/modules that have been deprecated for over a year have been deleted.
+
+## 0.3.3
+
+### New Features
+
+* Added a predicate `getValueConstant` to `AttributeArgument` that yields the argument value as an `Expr` when the value is a constant expression.
+* A new class predicate `MustFlowConfiguration::allowInterproceduralFlow` has been added to the `semmle.code.cpp.ir.dataflow.MustFlow` library. The new predicate can be overridden to disable interprocedural flow.
+* Added subclasses of `BuiltInOperations` for `__builtin_bit_cast`, `__builtin_shuffle`, `__has_unique_object_representations`, `__is_aggregate`, and `__is_assignable`.
+
+### Major Analysis Improvements
+
+* The IR dataflow library now includes flow through global variables. This enables new findings in many scenarios.
+
+## 0.3.2
+
+### Bug Fixes
+
+* Under certain circumstances a variable declaration that is not also a definition could be associated with a `Variable` that did not have the definition as a `VariableDeclarationEntry`. This is now fixed, and a unique `Variable` will exist that has both the declaration and the definition as a `VariableDeclarationEntry`.
+
+## 0.3.1
+
+### Minor Analysis Improvements
+
+* `AnalysedExpr::isNullCheck` and `AnalysedExpr::isValidCheck` have been updated to handle variable accesses on the left-hand side of the C++ logical "and", and variable declarations in conditions.
+
+## 0.3.0
+
+### Deprecated APIs
+
+* The `BarrierGuard` class has been deprecated. Such barriers and sanitizers can now instead be created using the new `BarrierGuard` parameterized module.
+
+### Bug Fixes
+
+* `UserType.getADeclarationEntry()` now yields all forward declarations when the user type is a `class`, `struct`, or `union`.
+
+## 0.2.3
+
+### New Features
+
+* An `isBraced` predicate was added to the `Initializer` class which holds when a C++ braced initializer was used in the initialization.
+
+## 0.2.2
+
+### Deprecated APIs
+
+ * The `AnalysedString` class in the `StringAnalysis` module has been replaced with `AnalyzedString`, to follow our style guide. The old name still exists as a deprecated alias.
+
+### New Features
+
+* A `getInitialization` predicate was added to the `ConstexprIfStmt`, `IfStmt`, and `SwitchStmt` classes that yields the C++17-style initializer of the `if` or `switch` statement when it exists.
+
 ## 0.2.1
 
 ## 0.2.0

cpp/ql/lib/change-notes/released/0.2.2.md

@@ -0,0 +1,9 @@
+## 0.2.2
+
+### Deprecated APIs
+
+ * The `AnalysedString` class in the `StringAnalysis` module has been replaced with `AnalyzedString`, to follow our style guide. The old name still exists as a deprecated alias.
+
+### New Features
+
+* A `getInitialization` predicate was added to the `ConstexprIfStmt`, `IfStmt`, and `SwitchStmt` classes that yields the C++17-style initializer of the `if` or `switch` statement when it exists.

cpp/ql/lib/change-notes/released/0.2.3.md

@@ -0,0 +1,5 @@
+## 0.2.3
+
+### New Features
+
+* An `isBraced` predicate was added to the `Initializer` class which holds when a C++ braced initializer was used in the initialization.

cpp/ql/lib/change-notes/released/0.3.0.md

@@ -0,0 +1,9 @@
+## 0.3.0
+
+### Deprecated APIs
+
+* The `BarrierGuard` class has been deprecated. Such barriers and sanitizers can now instead be created using the new `BarrierGuard` parameterized module.
+
+### Bug Fixes
+
+* `UserType.getADeclarationEntry()` now yields all forward declarations when the user type is a `class`, `struct`, or `union`.

cpp/ql/lib/change-notes/released/0.3.1.md

@@ -0,0 +1,5 @@
+## 0.3.1
+
+### Minor Analysis Improvements
+
+* `AnalysedExpr::isNullCheck` and `AnalysedExpr::isValidCheck` have been updated to handle variable accesses on the left-hand side of the C++ logical "and", and variable declarations in conditions.

cpp/ql/lib/change-notes/released/0.3.2.md

@@ -0,0 +1,5 @@
+## 0.3.2
+
+### Bug Fixes
+
+* Under certain circumstances a variable declaration that is not also a definition could be associated with a `Variable` that did not have the definition as a `VariableDeclarationEntry`. This is now fixed, and a unique `Variable` will exist that has both the declaration and the definition as a `VariableDeclarationEntry`.

cpp/ql/lib/change-notes/released/0.3.3.md

@@ -0,0 +1,11 @@
+## 0.3.3
+
+### New Features
+
+* Added a predicate `getValueConstant` to `AttributeArgument` that yields the argument value as an `Expr` when the value is a constant expression.
+* A new class predicate `MustFlowConfiguration::allowInterproceduralFlow` has been added to the `semmle.code.cpp.ir.dataflow.MustFlow` library. The new predicate can be overridden to disable interprocedural flow.
+* Added subclasses of `BuiltInOperations` for `__builtin_bit_cast`, `__builtin_shuffle`, `__has_unique_object_representations`, `__is_aggregate`, and `__is_assignable`.
+
+### Major Analysis Improvements
+
+* The IR dataflow library now includes flow through global variables. This enables new findings in many scenarios.

cpp/ql/lib/change-notes/released/0.3.4.md

@@ -0,0 +1,15 @@
+## 0.3.4
+
+### Deprecated APIs
+
+* Many classes/predicates/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.
+
+### New Features
+
+* Added support for getting the link targets of global and namespace variables.
+* Added a `BlockAssignExpr` class, which models a `memcpy`-like operation used in compiler generated copy/move constructors and assignment operations.
+
+### Minor Analysis Improvements
+
+* All deprecated predicates/classes/modules that have been deprecated for over a year have been deleted.

cpp/ql/lib/change-notes/released/0.3.5.md

@@ -0,0 +1 @@
+## 0.3.5

cpp/ql/lib/change-notes/released/0.4.0.md

@@ -0,0 +1,14 @@
+## 0.4.0
+
+### Deprecated APIs
+
+* Some classes/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.
+
+### New Features
+
+* Added subclasses of `BuiltInOperations` for `__is_same`, `__is_function`, `__is_layout_compatible`, `__is_pointer_interconvertible_base_of`, `__is_array`, `__array_rank`, `__array_extent`, `__is_arithmetic`, `__is_complete_type`, `__is_compound`, `__is_const`, `__is_floating_point`, `__is_fundamental`, `__is_integral`, `__is_lvalue_reference`, `__is_member_function_pointer`, `__is_member_object_pointer`, `__is_member_pointer`, `__is_object`, `__is_pointer`, `__is_reference`, `__is_rvalue_reference`, `__is_scalar`, `__is_signed`, `__is_unsigned`, `__is_void`, and `__is_volatile`.
+
+### Bug Fixes
+
+* Fixed an issue in the taint tracking analysis where implicit reads were not allowed by default in sinks or additional taint steps that used flow states.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.2.1
+lastReleaseVersion: 0.4.0

cpp/ql/lib/experimental/semmle/code/cpp/dataflow/ProductFlow.qll

@@ -0,0 +1,165 @@
+import experimental.semmle.code.cpp.ir.dataflow.DataFlow
+import experimental.semmle.code.cpp.ir.dataflow.DataFlow2
+
+module ProductFlow {
+  abstract class Configuration extends string {
+    bindingset[this]
+    Configuration() { any() }
+
+    /**
+     * Holds if `(source1, source2)` is a relevant data flow source.
+     *
+     * `source1` and `source2` must belong to the same callable.
+     */
+    predicate isSourcePair(DataFlow::Node source1, DataFlow::Node source2) { none() }
+
+    /**
+     * Holds if `(source1, source2)` is a relevant data flow source with initial states `state1`
+     * and `state2`, respectively.
+     *
+     * `source1` and `source2` must belong to the same callable.
+     */
+    predicate isSourcePair(
+      DataFlow::Node source1, string state1, DataFlow::Node source2, string state2
+    ) {
+      state1 = "" and
+      state2 = "" and
+      this.isSourcePair(source1, source2)
+    }
+
+    /**
+     * Holds if `(sink1, sink2)` is a relevant data flow sink.
+     *
+     * `sink1` and `sink2` must belong to the same callable.
+     */
+    predicate isSinkPair(DataFlow::Node sink1, DataFlow::Node sink2) { none() }
+
+    /**
+     * Holds if `(sink1, sink2)` is a relevant data flow sink with final states `state1`
+     * and `state2`, respectively.
+     *
+     * `sink1` and `sink2` must belong to the same callable.
+     */
+    predicate isSinkPair(
+      DataFlow::Node sink1, DataFlow::FlowState state1, DataFlow::Node sink2,
+      DataFlow::FlowState state2
+    ) {
+      state1 = "" and
+      state2 = "" and
+      this.isSinkPair(sink1, sink2)
+    }
+
+    predicate hasFlowPath(
+      DataFlow::PathNode source1, DataFlow2::PathNode source2, DataFlow::PathNode sink1,
+      DataFlow2::PathNode sink2
+    ) {
+      reachable(this, source1, source2, sink1, sink2)
+    }
+  }
+
+  private import Internal
+
+  module Internal {
+    class Conf1 extends DataFlow::Configuration {
+      Conf1() { this = "Conf1" }
+
+      override predicate isSource(DataFlow::Node source, string state) {
+        exists(Configuration conf | conf.isSourcePair(source, state, _, _))
+      }
+
+      override predicate isSink(DataFlow::Node sink, string state) {
+        exists(Configuration conf | conf.isSinkPair(sink, state, _, _))
+      }
+    }
+
+    class Conf2 extends DataFlow2::Configuration {
+      Conf2() { this = "Conf2" }
+
+      override predicate isSource(DataFlow::Node source, string state) {
+        exists(Configuration conf, DataFlow::Node source1 |
+          conf.isSourcePair(source1, _, source, state) and
+          any(Conf1 c).hasFlow(source1, _)
+        )
+      }
+
+      override predicate isSink(DataFlow::Node sink, string state) {
+        exists(Configuration conf, DataFlow::Node sink1 |
+          conf.isSinkPair(sink1, _, sink, state) and any(Conf1 c).hasFlow(_, sink1)
+        )
+      }
+    }
+  }
+
+  private predicate reachableInterprocEntry(
+    Configuration conf, DataFlow::PathNode source1, DataFlow2::PathNode source2,
+    DataFlow::PathNode node1, DataFlow2::PathNode node2
+  ) {
+    conf.isSourcePair(node1.getNode(), _, node2.getNode(), _) and
+    node1 = source1 and
+    node2 = source2
+    or
+    exists(
+      DataFlow::PathNode midEntry1, DataFlow2::PathNode midEntry2, DataFlow::PathNode midExit1,
+      DataFlow2::PathNode midExit2
+    |
+      reachableInterprocEntry(conf, source1, source2, midEntry1, midEntry2) and
+      interprocEdgePair(midExit1, midExit2, node1, node2) and
+      localPathStep1*(midEntry1, midExit1) and
+      localPathStep2*(midEntry2, midExit2)
+    )
+  }
+
+  private predicate localPathStep1(DataFlow::PathNode pred, DataFlow::PathNode succ) {
+    DataFlow::PathGraph::edges(pred, succ) and
+    pragma[only_bind_out](pred.getNode().getEnclosingCallable()) =
+      pragma[only_bind_out](succ.getNode().getEnclosingCallable())
+  }
+
+  private predicate localPathStep2(DataFlow2::PathNode pred, DataFlow2::PathNode succ) {
+    DataFlow2::PathGraph::edges(pred, succ) and
+    pragma[only_bind_out](pred.getNode().getEnclosingCallable()) =
+      pragma[only_bind_out](succ.getNode().getEnclosingCallable())
+  }
+
+  pragma[nomagic]
+  private predicate interprocEdge1(
+    Declaration predDecl, Declaration succDecl, DataFlow::PathNode pred1, DataFlow::PathNode succ1
+  ) {
+    DataFlow::PathGraph::edges(pred1, succ1) and
+    predDecl != succDecl and
+    pred1.getNode().getEnclosingCallable() = predDecl and
+    succ1.getNode().getEnclosingCallable() = succDecl
+  }
+
+  pragma[nomagic]
+  private predicate interprocEdge2(
+    Declaration predDecl, Declaration succDecl, DataFlow2::PathNode pred2, DataFlow2::PathNode succ2
+  ) {
+    DataFlow2::PathGraph::edges(pred2, succ2) and
+    predDecl != succDecl and
+    pred2.getNode().getEnclosingCallable() = predDecl and
+    succ2.getNode().getEnclosingCallable() = succDecl
+  }
+
+  private predicate interprocEdgePair(
+    DataFlow::PathNode pred1, DataFlow2::PathNode pred2, DataFlow::PathNode succ1,
+    DataFlow2::PathNode succ2
+  ) {
+    exists(Declaration predDecl, Declaration succDecl |
+      interprocEdge1(predDecl, succDecl, pred1, succ1) and
+      interprocEdge2(predDecl, succDecl, pred2, succ2)
+    )
+  }
+
+  private predicate reachable(
+    Configuration conf, DataFlow::PathNode source1, DataFlow2::PathNode source2,
+    DataFlow::PathNode sink1, DataFlow2::PathNode sink2
+  ) {
+    exists(DataFlow::PathNode mid1, DataFlow2::PathNode mid2 |
+      reachableInterprocEntry(conf, source1, source2, mid1, mid2) and
+      conf.isSinkPair(sink1.getNode(), _, sink2.getNode(), _) and
+      localPathStep1*(mid1, sink1) and
+      localPathStep2*(mid2, sink2)
+    )
+  }
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/DataFlow.qll

@@ -0,0 +1,26 @@
+/**
+ * Provides a library for local (intra-procedural) and global (inter-procedural)
+ * data flow analysis: deciding whether data can flow from a _source_ to a
+ * _sink_. This library differs from the one in `semmle.code.cpp.dataflow` in that
+ * this library uses the IR (Intermediate Representation) library, which provides
+ * a more precise semantic representation of the program, whereas the other dataflow
+ * library uses the more syntax-oriented ASTs. This library should provide more accurate
+ * results than the AST-based library in most scenarios.
+ *
+ * Unless configured otherwise, _flow_ means that the exact value of
+ * the source may reach the sink. We do not track flow across pointer
+ * dereferences or array indexing.
+ *
+ * To use global (interprocedural) data flow, extend the class
+ * `DataFlow::Configuration` as documented on that class. To use local
+ * (intraprocedural) data flow between expressions, call
+ * `DataFlow::localExprFlow`. For more general cases of local data flow, call
+ * `DataFlow::localFlow` or `DataFlow::localFlowStep` with arguments of type
+ * `DataFlow::Node`.
+ */
+
+import cpp
+
+module DataFlow {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/DataFlow2.qll

@@ -0,0 +1,16 @@
+/**
+ * Provides a `DataFlow2` module, which is a copy of the `DataFlow` module. Use
+ * this class when data-flow configurations must depend on each other. Two
+ * classes extending `DataFlow::Configuration` should never depend on each
+ * other, but one of them should instead depend on a
+ * `DataFlow2::Configuration`, a `DataFlow3::Configuration`, or a
+ * `DataFlow4::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.DataFlow` for the full documentation.
+ */
+
+import cpp
+
+module DataFlow2 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl2
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/DataFlow3.qll

@@ -0,0 +1,16 @@
+/**
+ * Provides a `DataFlow3` module, which is a copy of the `DataFlow` module. Use
+ * this class when data-flow configurations must depend on each other. Two
+ * classes extending `DataFlow::Configuration` should never depend on each
+ * other, but one of them should instead depend on a
+ * `DataFlow2::Configuration`, a `DataFlow3::Configuration`, or a
+ * `DataFlow4::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.DataFlow` for the full documentation.
+ */
+
+import cpp
+
+module DataFlow3 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl3
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/DataFlow4.qll

@@ -0,0 +1,16 @@
+/**
+ * Provides a `DataFlow4` module, which is a copy of the `DataFlow` module. Use
+ * this class when data-flow configurations must depend on each other. Two
+ * classes extending `DataFlow::Configuration` should never depend on each
+ * other, but one of them should instead depend on a
+ * `DataFlow2::Configuration`, a `DataFlow3::Configuration`, or a
+ * `DataFlow4::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.DataFlow` for the full documentation.
+ */
+
+import cpp
+
+module DataFlow4 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowImpl4
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/ResolveCall.qll

@@ -0,0 +1,23 @@
+/**
+ * Provides a predicate for non-contextual virtual dispatch and function
+ * pointer resolution.
+ */
+
+import cpp
+private import semmle.code.cpp.ir.ValueNumbering
+private import internal.DataFlowDispatch
+private import semmle.code.cpp.ir.IR
+
+/**
+ * Resolve potential target function(s) for `call`.
+ *
+ * If `call` is a call through a function pointer (`ExprCall`) or its target is
+ * a virtual member function, simple data flow analysis is performed in order
+ * to identify the possible target(s).
+ */
+Function resolveCall(Call call) {
+  exists(CallInstruction callInstruction |
+    callInstruction.getAst() = call and
+    result = viableCallable(callInstruction)
+  )
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/TaintTracking.qll

@@ -0,0 +1,23 @@
+/**
+ * Provides classes for performing local (intra-procedural) and
+ * global (inter-procedural) taint-tracking analyses.
+ *
+ * We define _taint propagation_ informally to mean that a substantial part of
+ * the information from the source is preserved at the sink. For example, taint
+ * propagates from `x` to `x + 100`, but it does not propagate from `x` to `x >
+ * 100` since we consider a single bit of information to be too little.
+ *
+ * To use global (interprocedural) taint tracking, extend the class
+ * `TaintTracking::Configuration` as documented on that class. To use local
+ * (intraprocedural) taint tracking between expressions, call
+ * `TaintTracking::localExprTaint`. For more general cases of local taint
+ * tracking, call `TaintTracking::localTaint` or
+ * `TaintTracking::localTaintStep` with arguments of type `DataFlow::Node`.
+ */
+
+import semmle.code.cpp.ir.dataflow.DataFlow
+import semmle.code.cpp.ir.dataflow.DataFlow2
+
+module TaintTracking {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.tainttracking1.TaintTrackingImpl
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/TaintTracking2.qll

@@ -0,0 +1,15 @@
+/**
+ * Provides a `TaintTracking2` module, which is a copy of the `TaintTracking`
+ * module. Use this class when data-flow configurations or taint-tracking
+ * configurations must depend on each other. Two classes extending
+ * `DataFlow::Configuration` should never depend on each other, but one of them
+ * should instead depend on a `DataFlow2::Configuration`, a
+ * `DataFlow3::Configuration`, or a `DataFlow4::Configuration`. The
+ * `TaintTracking::Configuration` class extends `DataFlow::Configuration`, and
+ * `TaintTracking2::Configuration` extends `DataFlow2::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.TaintTracking` for the full documentation.
+ */
+module TaintTracking2 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.tainttracking2.TaintTrackingImpl
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/TaintTracking3.qll

@@ -0,0 +1,15 @@
+/**
+ * Provides a `TaintTracking3` module, which is a copy of the `TaintTracking`
+ * module. Use this class when data-flow configurations or taint-tracking
+ * configurations must depend on each other. Two classes extending
+ * `DataFlow::Configuration` should never depend on each other, but one of them
+ * should instead depend on a `DataFlow2::Configuration`, a
+ * `DataFlow3::Configuration`, or a `DataFlow4::Configuration`. The
+ * `TaintTracking::Configuration` class extends `DataFlow::Configuration`, and
+ * `TaintTracking2::Configuration` extends `DataFlow2::Configuration`.
+ *
+ * See `semmle.code.cpp.ir.dataflow.TaintTracking` for the full documentation.
+ */
+module TaintTracking3 {
+  import experimental.semmle.code.cpp.ir.dataflow.internal.tainttracking3.TaintTrackingImpl
+}

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll

@@ -0,0 +1,273 @@
+private import cpp
+private import semmle.code.cpp.ir.IR
+private import experimental.semmle.code.cpp.ir.dataflow.DataFlow
+private import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowPrivate
+private import experimental.semmle.code.cpp.ir.dataflow.internal.DataFlowUtil
+private import DataFlowImplCommon as DataFlowImplCommon
+
+/**
+ * Gets a function that might be called by `call`.
+ */
+cached
+Function viableCallable(CallInstruction call) {
+  DataFlowImplCommon::forceCachingInSameStage() and
+  result = call.getStaticCallTarget()
+  or
+  // If the target of the call does not have a body in the snapshot, it might
+  // be because the target is just a header declaration, and the real target
+  // will be determined at run time when the caller and callee are linked
+  // together by the operating system's dynamic linker. In case a _unique_
+  // function with the right signature is present in the database, we return
+  // that as a potential callee.
+  exists(string qualifiedName, int nparams |
+    callSignatureWithoutBody(qualifiedName, nparams, call) and
+    functionSignatureWithBody(qualifiedName, nparams, result) and
+    strictcount(Function other | functionSignatureWithBody(qualifiedName, nparams, other)) = 1
+  )
+  or
+  // Virtual dispatch
+  result = call.(VirtualDispatch::DataSensitiveCall).resolve()
+}
+
+/**
+ * Provides virtual dispatch support compatible with the original
+ * implementation of `semmle.code.cpp.security.TaintTracking`.
+ */
+private module VirtualDispatch {
+  /** A call that may dispatch differently depending on the qualifier value. */
+  abstract class DataSensitiveCall extends DataFlowCall {
+    /**
+     * Gets the node whose value determines the target of this call. This node
+     * could be the qualifier of a virtual dispatch or the function-pointer
+     * expression in a call to a function pointer. What they have in common is
+     * that we need to find out which data flows there, and then it's up to the
+     * `resolve` predicate to stitch that information together and resolve the
+     * call.
+     */
+    abstract DataFlow::Node getDispatchValue();
+
+    /** Gets a candidate target for this call. */
+    abstract Function resolve();
+
+    /**
+     * Whether `src` can flow to this call.
+     *
+     * Searches backwards from `getDispatchValue()` to `src`. The `allowFromArg`
+     * parameter is true when the search is allowed to continue backwards into
+     * a parameter; non-recursive callers should pass `_` for `allowFromArg`.
+     */
+    predicate flowsFrom(DataFlow::Node src, boolean allowFromArg) {
+      src = this.getDispatchValue() and allowFromArg = true
+      or
+      exists(DataFlow::Node other, boolean allowOtherFromArg |
+        this.flowsFrom(other, allowOtherFromArg)
+      |
+        // Call argument
+        exists(DataFlowCall call, Position i |
+          other
+              .(DataFlow::ParameterNode)
+              .isParameterOf(pragma[only_bind_into](call).getStaticCallTarget(), i) and
+          src.(ArgumentNode).argumentOf(call, pragma[only_bind_into](pragma[only_bind_out](i)))
+        ) and
+        allowOtherFromArg = true and
+        allowFromArg = true
+        or
+        // Call return
+        exists(DataFlowCall call, ReturnKind returnKind |
+          other = getAnOutNode(call, returnKind) and
+          returnNodeWithKindAndEnclosingCallable(src, returnKind, call.getStaticCallTarget())
+        ) and
+        allowFromArg = false
+        or
+        // Local flow
+        DataFlow::localFlowStep(src, other) and
+        allowFromArg = allowOtherFromArg
+        or
+        // Flow from global variable to load.
+        exists(LoadInstruction load, GlobalOrNamespaceVariable var |
+          var = src.asVariable() and
+          other.asInstruction() = load and
+          addressOfGlobal(load.getSourceAddress(), var) and
+          // The `allowFromArg` concept doesn't play a role when `src` is a
+          // global variable, so we just set it to a single arbitrary value for
+          // performance.
+          allowFromArg = true
+        )
+        or
+        // Flow from store to global variable.
+        exists(StoreInstruction store, GlobalOrNamespaceVariable var |
+          var = other.asVariable() and
+          store = src.asInstruction() and
+          storeIntoGlobal(store, var) and
+          // Setting `allowFromArg` to `true` like in the base case means we
+          // treat a store to a global variable like the dispatch itself: flow
+          // may come from anywhere.
+          allowFromArg = true
+        )
+      )
+    }
+  }
+
+  pragma[noinline]
+  private predicate storeIntoGlobal(StoreInstruction store, GlobalOrNamespaceVariable var) {
+    addressOfGlobal(store.getDestinationAddress(), var)
+  }
+
+  /** Holds if `addressInstr` is an instruction that produces the address of `var`. */
+  private predicate addressOfGlobal(Instruction addressInstr, GlobalOrNamespaceVariable var) {
+    // Access directly to the global variable
+    addressInstr.(VariableAddressInstruction).getAstVariable() = var
+    or
+    // Access to a field on a global union
+    exists(FieldAddressInstruction fa |
+      fa = addressInstr and
+      fa.getObjectAddress().(VariableAddressInstruction).getAstVariable() = var and
+      fa.getField().getDeclaringType() instanceof Union
+    )
+  }
+
+  /**
+   * A ReturnNode with its ReturnKind and its enclosing callable.
+   *
+   * Used to fix a join ordering issue in flowsFrom.
+   */
+  pragma[noinline]
+  private predicate returnNodeWithKindAndEnclosingCallable(
+    ReturnNode node, ReturnKind kind, DataFlowCallable callable
+  ) {
+    node.getKind() = kind and
+    node.getEnclosingCallable() = callable
+  }
+
+  /** Call through a function pointer. */
+  private class DataSensitiveExprCall extends DataSensitiveCall {
+    DataSensitiveExprCall() { not exists(this.getStaticCallTarget()) }
+
+    override DataFlow::Node getDispatchValue() { result.asInstruction() = this.getCallTarget() }
+
+    override Function resolve() {
+      exists(FunctionInstruction fi |
+        this.flowsFrom(DataFlow::instructionNode(fi), _) and
+        result = fi.getFunctionSymbol()
+      ) and
+      (
+        this.getNumberOfArguments() <= result.getEffectiveNumberOfParameters() and
+        this.getNumberOfArguments() >= result.getEffectiveNumberOfParameters()
+        or
+        result.isVarargs()
+      )
+    }
+  }
+
+  /** Call to a virtual function. */
+  private class DataSensitiveOverriddenFunctionCall extends DataSensitiveCall {
+    DataSensitiveOverriddenFunctionCall() {
+      exists(this.getStaticCallTarget().(VirtualFunction).getAnOverridingFunction())
+    }
+
+    override DataFlow::Node getDispatchValue() { result.asInstruction() = this.getThisArgument() }
+
+    override MemberFunction resolve() {
+      exists(Class overridingClass |
+        this.overrideMayAffectCall(overridingClass, result) and
+        this.hasFlowFromCastFrom(overridingClass)
+      )
+    }
+
+    /**
+     * Holds if `this` is a virtual function call whose static target is
+     * overridden by `overridingFunction` in `overridingClass`.
+     */
+    pragma[noinline]
+    private predicate overrideMayAffectCall(Class overridingClass, MemberFunction overridingFunction) {
+      overridingFunction.getAnOverriddenFunction+() = this.getStaticCallTarget().(VirtualFunction) and
+      overridingFunction.getDeclaringType() = overridingClass
+    }
+
+    /**
+     * Holds if the qualifier of `this` has flow from an upcast from
+     * `derivedClass`.
+     */
+    pragma[noinline]
+    private predicate hasFlowFromCastFrom(Class derivedClass) {
+      exists(ConvertToBaseInstruction toBase |
+        this.flowsFrom(DataFlow::instructionNode(toBase), _) and
+        derivedClass = toBase.getDerivedClass()
+      )
+    }
+  }
+}
+
+/**
+ * Holds if `f` is a function with a body that has name `qualifiedName` and
+ * `nparams` parameter count. See `functionSignature`.
+ */
+private predicate functionSignatureWithBody(string qualifiedName, int nparams, Function f) {
+  functionSignature(f, qualifiedName, nparams) and
+  exists(f.getBlock())
+}
+
+/**
+ * Holds if the target of `call` is a function _with no definition_ that has
+ * name `qualifiedName` and `nparams` parameter count. See `functionSignature`.
+ */
+pragma[noinline]
+private predicate callSignatureWithoutBody(string qualifiedName, int nparams, CallInstruction call) {
+  exists(Function target |
+    target = call.getStaticCallTarget() and
+    not exists(target.getBlock()) and
+    functionSignature(target, qualifiedName, nparams)
+  )
+}
+
+/**
+ * Holds if `f` has name `qualifiedName` and `nparams` parameter count. This is
+ * an approximation of its signature for the purpose of matching functions that
+ * might be the same across link targets.
+ */
+private predicate functionSignature(Function f, string qualifiedName, int nparams) {
+  qualifiedName = f.getQualifiedName() and
+  nparams = f.getNumberOfParameters() and
+  not f.isStatic()
+}
+
+/**
+ * Holds if the set of viable implementations that can be called by `call`
+ * might be improved by knowing the call context.
+ */
+predicate mayBenefitFromCallContext(CallInstruction call, Function f) {
+  mayBenefitFromCallContext(call, f, _)
+}
+
+/**
+ * Holds if `call` is a call through a function pointer, and the pointer
+ * value is given as the `arg`'th argument to `f`.
+ */
+private predicate mayBenefitFromCallContext(
+  VirtualDispatch::DataSensitiveCall call, Function f, int arg
+) {
+  f = pragma[only_bind_out](call).getEnclosingCallable() and
+  exists(InitializeParameterInstruction init |
+    not exists(call.getStaticCallTarget()) and
+    init.getEnclosingFunction() = f and
+    call.flowsFrom(DataFlow::instructionNode(init), _) and
+    init.getParameter().getIndex() = arg
+  )
+}
+
+/**
+ * Gets a viable dispatch target of `call` in the context `ctx`. This is
+ * restricted to those `call`s for which a context might make a difference.
+ */
+Function viableImplInCallContext(CallInstruction call, CallInstruction ctx) {
+  result = viableCallable(call) and
+  exists(int i, Function f |
+    mayBenefitFromCallContext(pragma[only_bind_into](call), f, i) and
+    f = ctx.getStaticCallTarget() and
+    result = ctx.getArgument(i).getUnconvertedResultExpression().(FunctionAccess).getTarget()
+  )
+}
+
+/** Holds if arguments at position `apos` match parameters at position `ppos`. */
+pragma[inline]
+predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) { ppos = apos }