Merge pull request #10174 from github/release-prep/2.10.4

Release preparation for version 2.10.4

.bazelrc

@@ -0,0 +1,3 @@
+build --repo_env=CC=clang --repo_env=CXX=clang++ --copt="-std=c++17"
+
+try-import %workspace%/local.bazelrc

.bazelversion

@@ -0,0 +1 @@
+5.0.0

.codeqlmanifest.json

@@ -1,27 +0,0 @@
-{
-    "provide": [
-        "*/ql/src/qlpack.yml",
-        "*/ql/lib/qlpack.yml",
-        "*/ql/test/qlpack.yml",
-        "*/ql/examples/qlpack.yml",
-        "*/ql/consistency-queries/qlpack.yml",
-        "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
-        "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
-        "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml",
-        "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
-        "csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
-        "csharp/ql/campaigns/Solorigate/src/qlpack.yml",
-        "csharp/ql/campaigns/Solorigate/test/qlpack.yml",
-        "misc/legacy-support/*/qlpack.yml",
-        "misc/suite-helpers/qlpack.yml",
-        "ruby/extractor-pack/codeql-extractor.yml",
-        "ql/extractor-pack/codeql-extractor.yml"
-    ],
-    "versionPolicies": {
-      "default": {
-        "requireChangeNotes": true,
-        "committedPrereleaseSuffix": "dev",
-        "committedVersion": "nextPatchRelease"
-      }
-    }
-}

.devcontainer/devcontainer.json

@@ -3,6 +3,8 @@
     "rust-lang.rust",
     "bungcip.better-toml",
     "github.vscode-codeql",
+    "hbenl.vscode-test-explorer",
+    "ms-vscode.test-adapter-converter",
     "slevesque.vscode-zipexplorer"
   ],
   "settings": {

.devcontainer/swift/Dockerfile

@@ -0,0 +1,9 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.236.0/containers/cpp/.devcontainer/base.Dockerfile
+
+# [Choice] Debian / Ubuntu version (use Debian 11, Ubuntu 18.04/22.04 on local arm64/Apple Silicon): debian-11, debian-10, ubuntu-22.04, ubuntu-20.04, ubuntu-18.04
+FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-ubuntu-22.04
+
+USER root
+ADD root.sh /tmp/root.sh
+ADD update-codeql.sh /usr/local/bin/update-codeql
+RUN bash /tmp/root.sh && rm /tmp/root.sh

.devcontainer/swift/devcontainer.json

@@ -0,0 +1,25 @@
+{
+	"extensions": [
+		"github.vscode-codeql",
+		"hbenl.vscode-test-explorer",
+		"ms-vscode.test-adapter-converter",
+		"slevesque.vscode-zipexplorer",
+		"ms-vscode.cpptools"
+	],
+	"settings": {
+		"files.watcherExclude": {
+			"**/target/**": true
+		},
+		"codeQL.runningQueries.memory": 2048
+	},
+	"build": {
+		"dockerfile": "Dockerfile",
+	},
+	"runArgs": [
+		"--cap-add=SYS_PTRACE",
+		"--security-opt",
+		"seccomp=unconfined"
+	],
+	"remoteUser": "vscode",
+	"onCreateCommand": ".devcontainer/swift/user.sh"
+}

.devcontainer/swift/root.sh

@@ -0,0 +1,22 @@
+set -xe
+
+BAZELISK_VERSION=v1.12.0
+BAZELISK_DOWNLOAD_SHA=6b0bcb2ea15bca16fffabe6fda75803440375354c085480fe361d2cbf32501db
+
+apt-get update
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y install --no-install-recommends \
+    zlib1g-dev \
+    uuid-dev \
+    python3-distutils \
+    python3-pip \
+    bash-completion
+
+# Install Bazel
+curl -fSsL -o /usr/local/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-amd64
+echo "${BAZELISK_DOWNLOAD_SHA} */usr/local/bin/bazelisk" | sha256sum --check -
+chmod 0755 /usr/local/bin/bazelisk
+ln -s bazelisk /usr/local/bin/bazel
+
+# install latest codeql
+update-codeql

.devcontainer/swift/update-codeql.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+
+URL=https://github.com/github/codeql-cli-binaries/releases
+LATEST_VERSION=$(curl -L -s -H 'Accept: application/json' $URL/latest | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/')
+CURRENT_VERSION=v$(codeql version 2>/dev/null | sed -ne 's/.*release \([0-9.]*\)\./\1/p')
+if [[ $CURRENT_VERSION != $LATEST_VERSION ]]; then
+  if [[ $UID != 0 ]]; then
+    echo "update required, please run this script with sudo:"
+    echo "  sudo $0"
+    exit 1
+  fi
+  ZIP=$(mktemp codeql.XXXX.zip)
+  curl -fSqL -o $ZIP $URL/download/$LATEST_VERSION/codeql-linux64.zip
+  unzip -q $ZIP -d /opt
+  rm $ZIP
+  ln -sf /opt/codeql/codeql /usr/local/bin/codeql
+  echo installed version $LATEST_VERSION
+else
+  echo current version $CURRENT_VERSION is up-to-date
+fi

.devcontainer/swift/user.sh

@@ -0,0 +1,13 @@
+set -xe
+
+# add the workspace to the codeql search path
+mkdir -p /home/vscode/.config/codeql
+echo "--search-path /workspaces/codeql" > /home/vscode/.config/codeql/config
+
+# create a swift extractor pack with the current state
+cd /workspaces/codeql
+bazel run swift/create-extractor-pack
+
+#install and set up pre-commit
+python3 -m pip install pre-commit --no-warn-script-location
+$HOME/.local/bin/pre-commit install

.gitattributes

@@ -39,6 +39,7 @@
 *.py text
 *.lua text
 *.expected text
+*.go text
 
 # Explicitly set a bunch of known extensions to binary, because Git < 2.10 will treat
 # `* text=auto eol=lf` as `* text eol=lf`
@@ -52,6 +53,14 @@
 java/ql/test/stubs/**/*.java linguist-generated=true
 java/ql/test/experimental/stubs/**/*.java linguist-generated=true
 
+# Force git not to modify line endings for go or html files under the go/ql directory
+go/ql/**/*.go -text
+go/ql/**/*.html -text
+# Force git not to modify line endings for go dbschemes
+go/*.dbscheme -text
+# Preserve unusual line ending from codeql-go merge
+go/extractor/opencsv/CSVReader.java -text
+
 # For some languages, upgrade script testing references really old dbscheme
 # files from legacy upgrades that have CRLF line endings. Since upgrade
 # resolution relies on object hashes, we must suppress line ending conversion

.github/actions/fetch-codeql/action.yml

@@ -6,9 +6,9 @@ runs:
     - name: Fetch CodeQL
       shell: bash
       run: |
-        LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
-        gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
-        unzip -q -d "${RUNNER_TEMP}" codeql-linux64.zip
-        echo "${RUNNER_TEMP}/codeql" >> "${GITHUB_PATH}"
+        gh extension install github/gh-codeql
+        gh codeql set-channel nightly
+        gh codeql version
+        gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_PATH}"
       env:
         GITHUB_TOKEN: ${{ github.token }}

.github/dependabot.yml

@@ -16,3 +16,11 @@ updates:
     directory: "ruby/autobuilder"
     schedule:
       interval: "daily"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    ignore:
+      - dependency-name: '*'
+        update-types: ['version-update:semver-patch', 'version-update:semver-minor']

.github/labeler.yml

@@ -6,14 +6,23 @@
   - csharp/**/*
   - change-notes/**/*csharp*
 
+Go:
+  - go/**/*
+  - change-notes/**/*go.*
+
 Java:
-  - java/**/*
+  - any: [ 'java/**/*', '!java/kotlin-extractor/**/*', '!java/kotlin-explorer/**/*', '!java/ql/test/kotlin/**/*' ]
   - change-notes/**/*java.*
 
 JS:
-  - javascript/**/*
+  - any: [ 'javascript/**/*', '!javascript/ql/experimental/adaptivethreatmodeling/**/*' ]
   - change-notes/**/*javascript*
 
+Kotlin:
+  - java/kotlin-extractor/**/*
+  - java/kotlin-explorer/**/*
+  - java/ql/test/kotlin/**/*
+
 Python:
   - python/**/*
   - change-notes/**/*python*
@@ -22,10 +31,15 @@ Ruby:
   - ruby/**/*
   - change-notes/**/*ruby*
 
+Swift:
+  - swift/**/*
+  - change-notes/**/*swift*
+
 documentation:
   - "**/*.qhelp"
   - "**/*.md"
   - docs/**/*
 
-"QL-for-QL": 
-  - ql/**/*
+"QL-for-QL":
+  - ql/**/*
+  - .github/workflows/ql-for-ql*

.github/problem-matchers/codeql-query-format.json

@@ -0,0 +1,14 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "codeql-query-format",
+            "pattern": [
+                {
+                    "regexp": "^((.*) would change by autoformatting\\.)$",
+                    "file": 2,
+                    "message": 1
+                }
+            ]
+        }
+    ]
+}

.github/problem-matchers/codeql-syntax-check.json

@@ -0,0 +1,17 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "codeql-syntax-check",
+            "pattern": [
+                {
+                    "regexp": "^((ERROR|WARNING): .* \\((.*):(\\d+),(\\d+)-\\d+\\))$",
+                    "message": 1,
+                    "file": 3,
+                    "line": 4,
+                    "col": 5,
+                    "severity": 2
+                }
+            ]
+        }
+    ]
+}

.github/problem-matchers/codeql-test-run.json

@@ -0,0 +1,14 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "codeql-test-run",
+            "pattern": [
+                {
+                    "regexp": "(\\[.*\\] FAILED\\((RESULT|COMPILATION)\\) (.*))$",
+                    "file": 3,
+                    "message": 1
+                }
+            ]
+        }
+    ]
+}

.github/problem-matchers/make.json

@@ -0,0 +1,13 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "make",
+            "pattern": [
+                {
+                    "regexp": "^(make: \\*\\*\\* .*)$",
+                    "message": 1
+                }
+            ]
+        }
+    ]
+}

.github/workflows/check-change-note.yml

@@ -10,6 +10,7 @@ on:
       - "*/ql/lib/**/*.qll"
       - "!**/experimental/**"
       - "!ql/**"
+      - "!swift/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:

.github/workflows/check-qldoc.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "*/ql/lib/**"
       - .github/workflows/check-qldoc.yml
+      - .github/actions/fetch-codeql/action.yml
     branches:
       - main
       - "rc/*"
@@ -14,31 +15,30 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Install CodeQL
-        run: |
-          gh extension install github/gh-codeql
-          gh codeql set-channel nightly
-          gh codeql version
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 2
 
+      - name: Install CodeQL
+        uses: ./.github/actions/fetch-codeql
+
       - name: Check QLdoc coverage
         shell: bash
         run: |
           EXIT_CODE=0
-          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -o '^[a-z]*/ql/lib' || true; } | sort -u)"
+          # TODO: remove the swift exception from the regex when we fix generated QLdoc
+          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!swift)[a-z]*/ql/lib' || true; } | sort -u)"
           for pack_dir in ${changed_lib_packs}; do
             lang="${pack_dir%/ql/lib}"
-            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
+            codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
           done
           git checkout HEAD^
           for pack_dir in ${changed_lib_packs}; do
+            # When we add a new language, pack_dir would not exist in HEAD^.
+            # In this case the right thing to do is to skip the check.
+            [[ ! -d "${pack_dir}" ]] && continue
             lang="${pack_dir%/ql/lib}"
-            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
+            codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
             awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-current.txt" | sort -u > "${RUNNER_TEMP}/current-undocumented.txt"
             awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-baseline.txt" | sort -u > "${RUNNER_TEMP}/baseline-undocumented.txt"
             UNDOCUMENTED="$(grep -f <(comm -13 "${RUNNER_TEMP}/baseline-undocumented.txt" "${RUNNER_TEMP}/current-undocumented.txt") "${RUNNER_TEMP}/${lang}-current.txt" || true)"

.github/workflows/close-stale.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/stale@v3
+    - uses: actions/stale@v5
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Comment or remove the `Stale` label in order to avoid having this issue closed in 7 days.'

.github/workflows/codeql-analysis.yml

@@ -28,12 +28,12 @@ jobs:
 
     steps:
     - name: Setup dotnet
-      uses: actions/setup-dotnet@v1
+      uses: actions/setup-dotnet@v2
       with:
-        dotnet-version: 6.0.101
+        dotnet-version: 6.0.202
 
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
@@ -49,7 +49,7 @@ jobs:
     #  uses: github/codeql-action/autobuild@main
 
     # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
+    # 📚 https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
     # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
     #    and modify them (or add more) to build your code if your project

.github/workflows/csv-coverage-metrics.yml

@@ -12,13 +12,14 @@ on:
       - main
     paths:
       - ".github/workflows/csv-coverage-metrics.yml"
+      - ".github/actions/fetch-codeql/action.yml"
 
 jobs:
-  publish:
+  publish-java:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
       - name: Create empty database
@@ -31,13 +32,40 @@ jobs:
       - name: Capture coverage information
         run: |
           DATABASE="${{ runner.temp }}/java-database"
-          codeql database analyze --format=sarif-latest --output=metrics.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
-      - uses: actions/upload-artifact@v2
+          codeql database analyze --format=sarif-latest --output=metrics-java.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
+      - uses: actions/upload-artifact@v3
         with:
-          name: metrics.sarif
-          path: metrics.sarif
+          name: metrics-java.sarif
+          path: metrics-java.sarif
           retention-days: 20
       - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@v1
+        uses: github/codeql-action/upload-sarif@main
         with:
-          sarif_file: metrics.sarif
+          sarif_file: metrics-java.sarif
+  
+  publish-csharp:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Setup CodeQL
+        uses: ./.github/actions/fetch-codeql
+      - name: Create empty database
+        run: |
+          DATABASE="${{ runner.temp }}/csharp-database"
+          PROJECT="${{ runner.temp }}/csharp-project"
+          dotnet new classlib --language=C# --output="$PROJECT"
+          codeql database create "$DATABASE" --language=csharp --source-root="$PROJECT" --command 'dotnet build /t:rebuild csharp-project.csproj /p:UseSharedCompilation=false'
+      - name: Capture coverage information
+        run: |
+          DATABASE="${{ runner.temp }}/csharp-database"
+          codeql database analyze --format=sarif-latest --output=metrics-csharp.sarif -- "$DATABASE" ./csharp/ql/src/Metrics/Summaries/FrameworkCoverage.ql
+      - uses: actions/upload-artifact@v3
+        with:
+          name: metrics-csharp.sarif
+          path: metrics-csharp.sarif
+          retention-days: 20
+      - name: Upload SARIF file
+        uses: github/codeql-action/upload-sarif@main
+        with:
+          sarif_file: metrics-csharp.sarif

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -3,18 +3,20 @@ name: Check framework coverage changes
 on:
   pull_request:
     paths:
-      - '.github/workflows/csv-coverage-pr-comment.yml'
-      - '*/ql/src/**/*.ql'
-      - '*/ql/src/**/*.qll'
-      - '*/ql/lib/**/*.ql'
-      - '*/ql/lib/**/*.qll'
-      - 'misc/scripts/library-coverage/*.py'
+      - ".github/workflows/csv-coverage-pr-comment.yml"
+      - ".github/workflows/csv-coverage-pr-artifacts.yml"
+      - ".github/actions/fetch-codeql/action.yml"
+      - "*/ql/src/**/*.ql"
+      - "*/ql/src/**/*.qll"
+      - "*/ql/lib/**/*.ql"
+      - "*/ql/lib/**/*.qll"
+      - "misc/scripts/library-coverage/*.py"
       # input data files
-      - '*/documentation/library-coverage/cwe-sink.csv'
-      - '*/documentation/library-coverage/frameworks.csv'
+      - "*/documentation/library-coverage/cwe-sink.csv"
+      - "*/documentation/library-coverage/frameworks.csv"
     branches:
       - main
-      - 'rc/*'
+      - "rc/*"
 
 jobs:
   generate:
@@ -23,77 +25,72 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Dump GitHub context
-      env:
-        GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-      run: echo "$GITHUB_CONTEXT"
-    - name: Clone self (github/codeql) - MERGE
-      uses: actions/checkout@v2
-      with:
-        path: merge
-    - name: Clone self (github/codeql) - BASE
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 2
-        path: base
-    - run: |
-        git checkout HEAD^1
-        git log -1 --format='%H'
-      working-directory: base
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Generate CSV files on merge commit of the PR
-      run: |
-        echo "Running generator on merge"
-        PATH="$PATH:codeql-cli/codeql" python merge/misc/scripts/library-coverage/generate-report.py ci merge merge
-        mkdir out_merge
-        cp framework-coverage-*.csv out_merge/
-        cp framework-coverage-*.rst out_merge/
-    - name: Generate CSV files on base commit of the PR
-      run: |
-        echo "Running generator on base"
-        PATH="$PATH:codeql-cli/codeql" python base/misc/scripts/library-coverage/generate-report.py ci base base
-        mkdir out_base
-        cp framework-coverage-*.csv out_base/
-        cp framework-coverage-*.rst out_base/
-    - name: Generate diff of coverage reports
-      run: |
-        python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
-    - name: Upload CSV package list
-      uses: actions/upload-artifact@v2
-      with:
-        name: csv-framework-coverage-merge
-        path: |
-          out_merge/framework-coverage-*.csv
-          out_merge/framework-coverage-*.rst
-    - name: Upload CSV package list
-      uses: actions/upload-artifact@v2
-      with:
-        name: csv-framework-coverage-base
-        path: |
-          out_base/framework-coverage-*.csv
-          out_base/framework-coverage-*.rst
-    - name: Upload comparison results
-      uses: actions/upload-artifact@v2
-      with:
-        name: comparison
-        path: |
-          comparison.md
-    - name: Save PR number
-      run: |
-        mkdir -p pr
-        echo ${{ github.event.pull_request.number }} > pr/NR
-    - name: Upload PR number
-      uses: actions/upload-artifact@v2
-      with:
-        name: pr
-        path: pr/
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJSON(github.event) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Clone self (github/codeql) - MERGE
+        uses: actions/checkout@v3
+        with:
+          path: merge
+      - name: Clone self (github/codeql) - BASE
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+          path: base
+      - run: |
+          git checkout HEAD^1
+          git log -1 --format='%H'
+        working-directory: base
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Download CodeQL CLI
+        uses: ./merge/.github/actions/fetch-codeql
+      - name: Generate CSV files on merge commit of the PR
+        run: |
+          echo "Running generator on merge"
+          python merge/misc/scripts/library-coverage/generate-report.py ci merge merge
+          mkdir out_merge
+          cp framework-coverage-*.csv out_merge/
+          cp framework-coverage-*.rst out_merge/
+      - name: Generate CSV files on base commit of the PR
+        run: |
+          echo "Running generator on base"
+          python base/misc/scripts/library-coverage/generate-report.py ci base base
+          mkdir out_base
+          cp framework-coverage-*.csv out_base/
+          cp framework-coverage-*.rst out_base/
+      - name: Generate diff of coverage reports
+        run: |
+          python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
+      - name: Upload CSV package list
+        uses: actions/upload-artifact@v3
+        with:
+          name: csv-framework-coverage-merge
+          path: |
+            out_merge/framework-coverage-*.csv
+            out_merge/framework-coverage-*.rst
+      - name: Upload CSV package list
+        uses: actions/upload-artifact@v3
+        with:
+          name: csv-framework-coverage-base
+          path: |
+            out_base/framework-coverage-*.csv
+            out_base/framework-coverage-*.rst
+      - name: Upload comparison results
+        uses: actions/upload-artifact@v3
+        with:
+          name: comparison
+          path: |
+            comparison.md
+      - name: Save PR number
+        run: |
+          mkdir -p pr
+          echo ${{ github.event.pull_request.number }} > pr/NR
+      - name: Upload PR number
+        uses: actions/upload-artifact@v3
+        with:
+          name: pr
+          path: pr/

.github/workflows/csv-coverage-pr-comment.yml

@@ -20,9 +20,9 @@ jobs:
         GITHUB_CONTEXT: ${{ toJSON(github.event) }}
       run: echo "$GITHUB_CONTEXT"
     - name: Clone self (github/codeql)
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.8
 

.github/workflows/csv-coverage-timeseries.yml

@@ -5,38 +5,29 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v2
-      with:
-        path: script
-    - name: Clone self (github/codeql) for analysis
-      uses: actions/checkout@v2
-      with:
-        path: codeqlModels
-        fetch-depth: 0
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Build modeled package list
-      run: |
-        CLI=$(realpath "codeql-cli/codeql")
-        echo $CLI
-        PATH="$PATH:$CLI" python script/misc/scripts/library-coverage/generate-timeseries.py codeqlModels
-    - name: Upload timeseries CSV
-      uses: actions/upload-artifact@v2
-      with:
-        name: framework-coverage-timeseries
-        path: framework-coverage-timeseries-*.csv
-
+      - name: Clone self (github/codeql)
+        uses: actions/checkout@v3
+        with:
+          path: script
+      - name: Clone self (github/codeql) for analysis
+        uses: actions/checkout@v3
+        with:
+          path: codeqlModels
+          fetch-depth: 0
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Download CodeQL CLI
+        uses: ./script/.github/actions/fetch-codeql
+      - name: Build modeled package list
+        run: |
+          python script/misc/scripts/library-coverage/generate-timeseries.py codeqlModels
+      - name: Upload timeseries CSV
+        uses: actions/upload-artifact@v3
+        with:
+          name: framework-coverage-timeseries
+          path: framework-coverage-timeseries-*.csv

.github/workflows/csv-coverage-update.yml

@@ -12,33 +12,27 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Dump GitHub context
-      env:
-        GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-      run: echo "$GITHUB_CONTEXT"
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v2
-      with:
-        path: ql
-        fetch-depth: 0
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJSON(github.event) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Clone self (github/codeql)
+        uses: actions/checkout@v3
+        with:
+          path: ql
+          fetch-depth: 0
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Download CodeQL CLI
+        uses: ./ql/.github/actions/fetch-codeql
+      - name: Generate coverage files
+        run: |
+          python ql/misc/scripts/library-coverage/generate-report.py ci ql ql
 
-    - name: Generate coverage files
-      run: |
-        PATH="$PATH:codeql-cli/codeql" python ql/misc/scripts/library-coverage/generate-report.py ci ql ql
-
-    - name: Create pull request with changes
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-        python ql/misc/scripts/library-coverage/create-pr.py ql "$GITHUB_REPOSITORY"
+      - name: Create pull request with changes
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python ql/misc/scripts/library-coverage/create-pr.py ql "$GITHUB_REPOSITORY"

.github/workflows/csv-coverage.yml

@@ -4,46 +4,39 @@ on:
   workflow_dispatch:
     inputs:
       qlModelShaOverride:
-        description: 'github/codeql repo SHA used for looking up the CSV models'
+        description: "github/codeql repo SHA used for looking up the CSV models"
         required: false
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v2
-      with:
-        path: script
-    - name: Clone self (github/codeql) for analysis
-      uses: actions/checkout@v2
-      with:
-        path: codeqlModels
-        ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Build modeled package list
-      run: |
-        PATH="$PATH:codeql-cli/codeql" python script/misc/scripts/library-coverage/generate-report.py ci codeqlModels script
-    - name: Upload CSV package list
-      uses: actions/upload-artifact@v2
-      with:
-        name: framework-coverage-csv
-        path: framework-coverage-*.csv
-    - name: Upload RST package list
-      uses: actions/upload-artifact@v2
-      with:
-        name: framework-coverage-rst
-        path: framework-coverage-*.rst
-
+      - name: Clone self (github/codeql)
+        uses: actions/checkout@v3
+        with:
+          path: script
+      - name: Clone self (github/codeql) for analysis
+        uses: actions/checkout@v3
+        with:
+          path: codeqlModels
+          ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Download CodeQL CLI
+        uses: ./script/.github/actions/fetch-codeql
+      - name: Build modeled package list
+        run: |
+          python script/misc/scripts/library-coverage/generate-report.py ci codeqlModels script
+      - name: Upload CSV package list
+        uses: actions/upload-artifact@v3
+        with:
+          name: framework-coverage-csv
+          path: framework-coverage-*.csv
+      - name: Upload RST package list
+        uses: actions/upload-artifact@v3
+        with:
+          name: framework-coverage-rst
+          path: framework-coverage-*.rst

.github/workflows/go-tests.yml

@@ -0,0 +1,114 @@
+name: "Go: Run Tests"
+on:
+  pull_request:
+    paths:
+      - "go/**"
+      - .github/workflows/go-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+jobs:
+  test-linux:
+    name: Test Linux (Ubuntu)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Check that all QL and Go code is autoformatted
+        run: |
+          cd go
+          make check-formatting
+
+      - name: Compile qhelp files to markdown
+        run: |
+          cd go
+          env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown
+
+      - name: Upload qhelp markdown
+        uses: actions/upload-artifact@v2
+        with:
+          name: qhelp-markdown
+          path: go/qhelp-out/**/*.md
+
+      - name: Test
+        run: |
+          cd go
+          make test
+
+  test-mac:
+    name: Test MacOS
+    runs-on: macos-latest
+    steps:
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Test
+        run: |
+          cd go
+          make test
+
+  test-win:
+    name: Test Windows
+    runs-on: windows-2019
+    steps:
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Test
+        run: |
+          cd go
+          make test

.github/workflows/js-ml-tests.yml

@@ -5,6 +5,8 @@ on:
     paths:
       - "javascript/ql/experimental/adaptivethreatmodeling/**"
       - .github/workflows/js-ml-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
@@ -12,6 +14,9 @@ on:
     paths:
       - "javascript/ql/experimental/adaptivethreatmodeling/**"
       - .github/workflows/js-ml-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+  workflow_dispatch:
 
 defaults:
   run:
@@ -22,7 +27,7 @@ jobs:
     name: Check QL formatting
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - uses: ./.github/actions/fetch-codeql
 
@@ -35,7 +40,7 @@ jobs:
     name: Check QL compilation
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - uses: ./.github/actions/fetch-codeql
 
@@ -59,7 +64,7 @@ jobs:
     name: Run QL tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - uses: ./.github/actions/fetch-codeql
 

.github/workflows/labeler.yml

@@ -4,8 +4,11 @@ on:
 
 jobs:
   triage:
+    permissions:
+      contents: read
+      pull-requests: write
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/labeler@v2
+    - uses: actions/labeler@v4
       with:
         repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/mad_modelDiff.yml

@@ -27,12 +27,12 @@ jobs:
         slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
     steps:
       - name: Clone github/codeql from PR
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         if: github.event.pull_request
         with:
           path: codeql-pr
       - name: Clone github/codeql from main
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           path: codeql-main
           ref: main
@@ -61,7 +61,7 @@ jobs:
             DATABASE=$2
             cd codeql-$QL_VARIANT
             SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
+            python java/ql/src/utils/model-generator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $MODELS/${SHORTNAME}.qll
             mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
             cd ..
           }
@@ -91,12 +91,12 @@ jobs:
             name="diff_${basename/_main.qll/""}"
             (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
           done
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: models
           path: tmp-models/*.qll
           retention-days: 20
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: diffs
           path: tmp-models/*.html

.github/workflows/mad_regenerate-models.yml

@@ -9,6 +9,7 @@ on:
       - main
     paths:
       - ".github/workflows/mad_regenerate-models.yml"
+      - ".github/actions/fetch-codeql/action.yml"
 
 jobs:
   regenerate-models:
@@ -20,17 +21,17 @@ jobs:
         ref: ["placeholder"]
         include:
           - slug: "apache/commons-io"
-            ref: "8985de8fe74f6622a419b37a6eed0dbc484dc128"
+            ref: "13258ce2d07aa0e764bbaa8020af4dcd3a02a620"
         exclude:
           - slug: "placeholder"
             ref: "placeholder"
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Setup CodeQL binaries
         uses: ./.github/actions/fetch-codeql
       - name: Clone repositories
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           path: repos/${{ matrix.ref }}
           ref: ${{ matrix.ref }}
@@ -55,7 +56,7 @@ jobs:
           find java -name "*.qll" -print0 | xargs -0 git add
           git status
           git diff --cached > models.patch
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: patch
           path: models.patch

.github/workflows/post-pr-comment.yml

@@ -1,12 +1,17 @@
-name: Post pull-request comment
+# This workflow is the second part of the process described in
+# .github/workflows/qhelp-pr-preview.yml
+# See that file for more info.
+
+name: Post PR comment
 on:
   workflow_run:
-    workflows: ["Query help preview"]
+    workflows: [Render QHelp changes]
     types:
       - completed
 
 permissions:
   pull-requests: write
+  actions: read
 
 jobs:
   post_comment:
@@ -17,15 +22,53 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ github.token }}
           WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
-      - run: |
-          PR="$(grep -o '^[0-9]\+$' pr.txt)"
+
+      - name: Check that PR SHA matches workflow SHA
+        run: |
+          PR="$(grep -o '^[0-9]\+$' pr_number.txt)"
           PR_HEAD_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/pulls/${PR}" --jq .head.sha)"
           # Check that the pull-request head SHA matches the head SHA of the workflow run
           if [ "${WORKFLOW_RUN_HEAD_SHA}" != "${PR_HEAD_SHA}" ]; then
             echo "PR head SHA ${PR_HEAD_SHA} does not match workflow_run event SHA ${WORKFLOW_RUN_HEAD_SHA}. Stopping." 1>&2
             exit 1
           fi
-          gh pr comment "${PR}" --repo "${GITHUB_REPOSITORY}" -F comment.txt
         env:
           GITHUB_TOKEN: ${{ github.token }}
           WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_commit.id }}
+
+      - name: Create or update comment
+        run: |
+          COMMENT_PREFIX="QHelp previews"
+          COMMENT_AUTHOR="github-actions[bot]"
+          PR_NUMBER="$(grep -o '^[0-9]\+$' pr_number.txt)"
+
+          # If there is no existing comment, comment_id.txt will contain just a
+          # newline (due to jq & gh behaviour). This will cause grep to fail, so
+          # we catch that.
+          RAW_COMMENT_ID=$(grep -o '^[0-9]\+$' comment_id.txt || true)
+
+          if [ $RAW_COMMENT_ID ]
+          then
+            # Fetch existing comment, and validate:
+            # - comment belongs to the PR with number $PR_NUMBER
+            # - comment starts with the expected prefix ("QHelp previews")
+            # - comment author is github-actions[bot]
+            FILTER='select(.issue_url | endswith($repo+"/issues/"+$pr))
+                  | select(.body | startswith($prefix))
+                  | select(.user.login == $author)
+                  | .id'
+            COMMENT_ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${RAW_COMMENT_ID}" | jq --arg repo "${GITHUB_REPOSITORY}" --arg pr "${PR_NUMBER}" --arg prefix "${COMMENT_PREFIX}" --arg author "${COMMENT_AUTHOR}" "${FILTER}")
+            if [ $COMMENT_ID ]
+            then
+              # Update existing comment
+              jq --rawfile body comment_body.txt '{"body":$body}' -n | gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${COMMENT_ID}" -X PATCH --input -
+            else
+              echo "Comment ${RAW_COMMENT_ID} did not pass validations: not editing." >&2
+              exit 1
+            fi
+          else
+            # Create new comment
+            jq --rawfile body comment_body.txt '{"body":$body}' -n | gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -X POST --input -
+          fi
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/qhelp-pr-preview.yml

@@ -1,7 +1,25 @@
-name: Query help preview
+# This workflow checks for any changes in .qhelp files in pull requests.
+# For any changed files, it renders them to markdown in a file called `comment_body.txt`.
+# It then checks if there's an existing comment on the pull request generated by
+# this workflow, and writes the comment ID to `comment_id.txt`.
+# It also writes the PR number to `pr_number.txt`.
+# These three files are uploaded as an artifact.
+
+# When this workflow completes, the workflow "Post PR comment" runs.
+# It downloads the artifact and adds a comment to the PR with the rendered
+# QHelp.
+
+# The task is split like this because creating PR comments requires extra
+# permissions that we don't want to expose to PRs from external forks.
+
+# For more info see:
+# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_run
+# https://docs.github.com/en/actions/security-guides/automatic-token-authentication#permissions-for-the-github_token
+name: Render QHelp changes
 
 permissions:
   contents: read
+  pull-requests: read
 
 on:
   pull_request:
@@ -15,13 +33,17 @@ jobs:
   qhelp:
     runs-on: ubuntu-latest
     steps:
-      - run: echo "${{  github.event.number }}" > pr.txt
-      - uses: actions/upload-artifact@v2
+      - run: echo "${PR_NUMBER}" > pr_number.txt
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+      - uses: actions/upload-artifact@v3
         with:
           name: comment
-          path: pr.txt
+          path: pr_number.txt
+          if-no-files-found: error
           retention-days: 1
-      - uses: actions/checkout@v2
+
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 2
           persist-credentials: false
@@ -36,7 +58,7 @@ jobs:
       - name: QHelp preview
         run: |
           EXIT_CODE=0
-          echo "QHelp previews:" > comment.txt
+          echo "QHelp previews:" > comment_body.txt
           while read -r -d $'\0' path; do
             if [ ! -f "${path}" ]; then
                exit 1
@@ -52,12 +74,29 @@ jobs:
                echo '```'
             fi
             echo "</details>"
-          done < "${RUNNER_TEMP}/paths.txt" >> comment.txt
+          done < "${RUNNER_TEMP}/paths.txt" >> comment_body.txt
           exit "${EXIT_CODE}"
 
       - if: always()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: comment
-          path: comment.txt
+          path: comment_body.txt
+          if-no-files-found: error
+          retention-days: 1
+
+      - name: Save ID of existing QHelp comment (if it exists)
+        run: |
+          # Find the latest comment starting with "QHelp previews"
+          COMMENT_PREFIX="QHelp previews"
+          gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate | jq --arg prefix "${COMMENT_PREFIX}" '[.[] | select(.body|startswith($prefix)) | .id] | max' > comment_id.txt
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.number }}
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: comment
+          path: comment_id.txt
+          if-no-files-found: error
           retention-days: 1

.github/workflows/ql-for-ql-build.yml

@@ -10,13 +10,14 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  queries:
-    runs-on: ubuntu-latest
+  analyze:
+    runs-on: ubuntu-latest-xl
     steps:
-      - uses: actions/checkout@v2
+      ### Build the queries ###
+      - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: javascript # does not matter
       - name: Get CodeQL version
@@ -26,39 +27,39 @@ jobs:
         shell: bash
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Cache queries
-        id: cache-queries
-        uses: actions/cache@v2
+      - name: Cache entire pack
+        id: cache-pack
+        uses: actions/cache@v3
         with:
-          path: ${{ runner.temp }}/query-pack.zip
-          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
+          path: ${{ runner.temp }}/pack
+          key: ${{ runner.os }}-pack-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
+      - name: Cache queries
+        if: steps.cache-pack.outputs.cache-hit != 'true'
+        id: cache-queries
+        uses: actions/cache@v3
+        with:
+          path: ${{ runner.temp }}/queries
+          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
       - name: Build query pack
-        if: steps.cache-queries.outputs.cache-hit != 'true'
+        if: steps.cache-queries.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         run: |
           cd ql/ql/src
-          "${CODEQL}" pack create
-          cd .codeql/pack/codeql/ql/0.0.0
-          zip "${PACKZIP}" -r .
+          "${CODEQL}" pack create -j 16
+          mv .codeql/pack/codeql/ql/0.0.0 ${{ runner.temp }}/queries
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-          PACKZIP: ${{ runner.temp }}/query-pack.zip
-      - name: Upload query pack
-        uses: actions/upload-artifact@v2
-        with:
-          name: query-pack-zip
-          path: ${{ runner.temp }}/query-pack.zip
-
-  extractors:
-    strategy:
-      fail-fast: false
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
+      - name: Move cache queries to pack
+        if: steps.cache-pack.outputs.cache-hit != 'true'
+        run: |
+          cp -r ${{ runner.temp }}/queries ${{ runner.temp }}/pack
+        env:
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+      
+      ### Build the extractor ###
       - name: Cache entire extractor
+        if: steps.cache-pack.outputs.cache-hit != 'true'
         id: cache-extractor
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             ql/target/release/ql-autobuilder
@@ -67,8 +68,8 @@ jobs:
             ql/target/release/ql-extractor.exe
           key: ${{ runner.os }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
       - name: Cache cargo
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        uses: actions/cache@v2
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        uses: actions/cache@v3
         with:
           path: |
             ~/.cargo/registry
@@ -76,124 +77,87 @@ jobs:
             ql/target
           key: ${{ runner.os }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Check formatting
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         run: cd ql; cargo fmt --all -- --check
       - name: Build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         run: cd ql; cargo build --verbose
       - name: Run tests
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         run: cd ql; cargo test --verbose
       - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         run: cd ql; cargo build --release
       - name: Generate dbscheme
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
         run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v2
-        with:
-          name: extractor-ubuntu-latest
-          path: |
-            ql/target/release/ql-autobuilder
-            ql/target/release/ql-autobuilder.exe
-            ql/target/release/ql-extractor
-            ql/target/release/ql-extractor.exe
-          retention-days: 1
-  package:
-    runs-on: ubuntu-latest
 
-    needs:
-      - extractors
-      - queries
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
-        with:
-          name: query-pack-zip
-          path: query-pack-zip
-      - uses: actions/download-artifact@v2
-        with:
-          name: extractor-ubuntu-latest
-          path: linux64
-      - run: |
-          unzip query-pack-zip/*.zip -d pack
-          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats pack/
-          mkdir -p pack/tools/linux64
-          if [[ -f linux64/ql-autobuilder ]]; then
-            cp linux64/ql-autobuilder pack/tools/linux64/autobuilder
-            chmod +x pack/tools/linux64/autobuilder
-          fi
-          if [[ -f linux64/ql-extractor ]]; then
-            cp linux64/ql-extractor pack/tools/linux64/extractor
-            chmod +x pack/tools/linux64/extractor
-          fi
-          cd pack
-          zip -rq ../codeql-ql.zip .
-      - uses: actions/upload-artifact@v2
-        with:
-          name: codeql-ql-pack
-          path: codeql-ql.zip
-          retention-days: 1
-  analyze:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        folder: [cpp, csharp, java, javascript, python, ql, ruby]
-
-    needs:
-      - package
-
-    steps:
-      - name: Download pack
-        uses: actions/download-artifact@v2
-        with:
-          name: codeql-ql-pack
-          path: ${{ runner.temp }}/codeql-ql-pack-artifact
-
-      - name: Prepare pack
+      ### Package the queries and extractor ###
+      - name: Package pack
+        if: steps.cache-pack.outputs.cache-hit != 'true'
         run: |
-          unzip "${PACK_ARTIFACT}/*.zip" -d "${PACK}"
+          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats ${PACK}/
+          mkdir -p ${PACK}/tools/linux64
+          cp ql/target/release/ql-autobuilder  ${PACK}/tools/linux64/autobuilder
+          cp ql/target/release/ql-extractor ${PACK}/tools/linux64/extractor
+          chmod +x ${PACK}/tools/linux64/autobuilder
+          chmod +x ${PACK}/tools/linux64/extractor
         env:
-          PACK_ARTIFACT: ${{ runner.temp }}/codeql-ql-pack-artifact
           PACK: ${{ runner.temp }}/pack
+
+      ### Run the analysis ###
       - name: Hack codeql-action options
         run: |
-          JSON=$(jq -nc --arg pack "${PACK}" '.resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
+          JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
           echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
         env:
           PACK: ${{ runner.temp }}/pack
 
-      - name: Checkout repository
-        uses: actions/checkout@v2
       - name: Create CodeQL config file
         run: |
-          echo "paths:" > ${CONF}
-          echo "  - ${FOLDER}" >> ${CONF}
           echo "paths-ignore:" >> ${CONF}
-          echo "  - ql/ql/test" >> ${CONF}
+          echo "  - ql/ql/test" >> ${CONF} 
+          echo "  - \"*/ql/lib/upgrades/\"" >> ${CONF} 
+          echo "disable-default-queries: true" >> ${CONF}
+          echo "queries:" >> ${CONF}
+          echo "  - uses: ./ql/ql/src/codeql-suites/ql-code-scanning.qls" >> ${CONF}
           echo "Config file: "
           cat ${CONF}
         env: 
           CONF: ./ql-for-ql-config.yml
-          FOLDER: ${{ matrix.folder }}
-
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: ql
           db-location: ${{ runner.temp }}/db
           config-file: ./ql-for-ql-config.yml
+      - name: Move pack cache
+        run: |
+          cp -r ${PACK}/.cache ql/ql/src/.cache
+        env:
+          PACK: ${{ runner.temp }}/pack
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@erik-krogh/ql
+        uses: github/codeql-action/analyze@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with: 
-          category: "ql-for-ql-${{ matrix.folder }}"
+          category: "ql-for-ql"
       - name: Copy sarif file to CWD
-        run: cp ../results/ql.sarif ./${{ matrix.folder }}.sarif
+        run: cp ../results/ql.sarif ./ql-for-ql.sarif
+      - name: Fixup the $scema in sarif  # Until https://github.com/microsoft/sarif-vscode-extension/pull/436/ is part in a stable release
+        run: |
+          sed -i 's/\$schema.*/\$schema": "https:\/\/raw.githubusercontent.com\/oasis-tcs\/sarif-spec\/master\/Schemata\/sarif-schema-2.1.0",/' ql-for-ql.sarif 
       - name: Sarif as artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.folder }}.sarif
-          path: ${{ matrix.folder }}.sarif
-
+          name: ql-for-ql.sarif
+          path: ql-for-ql.sarif
+      - name: Split out the sarif file into langs
+        run: |
+          mkdir split-sarif
+          node ./ql/scripts/split-sarif.js ql-for-ql.sarif split-sarif
+      - name: Upload langs as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ql-for-ql-langs
+          path: split-sarif
+          retention-days: 1

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -19,17 +19,16 @@ jobs:
       matrix:
         repo:
           - github/codeql
-          - github/codeql-go
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: javascript # does not matter
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: |
             ~/.cargo/registry
@@ -37,11 +36,11 @@ jobs:
             ql/target
           key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Build Extractor
-        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
+        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./scripts/create-extractor-pack.sh
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: ${{ matrix.repo }}
           path: ${{ github.workspace }}/repo
@@ -60,7 +59,7 @@ jobs:
           "${CODEQL}" dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ql"
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: measurements
           path: stats
@@ -70,15 +69,15 @@ jobs:
     runs-on: ubuntu-latest
     needs: measure
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
         with:
           name: measurements
           path: stats
       - run: |
           python -m pip install --user lxml
-          find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ql/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
-      - uses: actions/upload-artifact@v2
+          find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ruby/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
+      - uses: actions/upload-artifact@v3
         with:
           name: ql.dbscheme.stats
           path: ql/ql/src/ql.dbscheme.stats

.github/workflows/ql-for-ql-tests.yml

@@ -5,10 +5,12 @@ on:
     branches: [main]
     paths:
       - "ql/**"
+      - codeql-workspace.yml
   pull_request:
     branches: [main]
     paths:
       - "ql/**"
+      - codeql-workspace.yml
 
 env:
   CARGO_TERM_COLOR: always
@@ -17,13 +19,13 @@ jobs:
   qltest:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
         with:
           languages: javascript # does not matter
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: |
             ~/.cargo/registry
@@ -34,7 +36,7 @@ jobs:
         run: |
           cd ql;
           codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
-          env "PATH=$PATH:$codeqlpath" ./create-extractor-pack.sh
+          env "PATH=$PATH:$codeqlpath" ./scripts/create-extractor-pack.sh
       - name: Run QL tests
         run: |
           "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test
@@ -42,7 +44,7 @@ jobs:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Check QL formatting
         run: |
-          find ql/ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
+          find ql/ql/src "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Check QL compilation

.github/workflows/query-list.yml

@@ -5,9 +5,12 @@ on:
     branches:
      - main
      - 'rc/**'
+    tags:
+     - 'codeql-cli/*'
   pull_request:
     paths:
       - '.github/workflows/query-list.yml'
+      - '.github/actions/fetch-codeql/action.yml'
       - 'misc/scripts/generate-code-scanning-query-list.py'
 
 jobs:
@@ -17,33 +20,21 @@ jobs:
 
     steps:
     - name: Clone self (github/codeql)
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         path: codeql 
-    - name: Clone github/codeql-go
-      uses: actions/checkout@v2
-      with:
-        repository: 'github/codeql-go'
-        path: codeql-go
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.8
     - name: Download CodeQL CLI
-      uses: dsaltares/fetch-gh-release-asset@aa37ae5c44d3c9820bc12fe675e8670ecd93bd1c
-      with:
-        repo: "github/codeql-cli-binaries"
-        version: "latest"
-        file: "codeql-linux64.zip"
-        token: ${{ secrets.GITHUB_TOKEN }}
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
+      # Look under the `codeql` directory, as this is where we checked out the `github/codeql` repo
+      uses: ./codeql/.github/actions/fetch-codeql
     - name: Build code scanning query list
       run: |
-        PATH="$PATH:codeql-cli/codeql" python codeql/misc/scripts/generate-code-scanning-query-list.py > code-scanning-query-list.csv
+        python codeql/misc/scripts/generate-code-scanning-query-list.py > code-scanning-query-list.csv
     - name: Upload code scanning query list
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: code-scanning-query-list
         path: code-scanning-query-list.csv
-    

.github/workflows/ruby-build.yml

@@ -5,6 +5,8 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-build.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
@@ -12,6 +14,8 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-build.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
@@ -38,13 +42,13 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install GNU tar
         if: runner.os == 'macOS'
         run: |
           brew install gnu-tar
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: |
             ~/.cargo/registry
@@ -62,17 +66,17 @@ jobs:
       - name: Generate dbscheme
         if: ${{ matrix.os == 'ubuntu-latest' }}
         run: target/release/ruby-generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           name: ruby.dbscheme
           path: ruby/ql/lib/ruby.dbscheme
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           name: TreeSitter.qll
           path: ruby/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: extractor-${{ matrix.os }}
           path: |
@@ -86,23 +90,18 @@ jobs:
     env:
       CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Fetch CodeQL
-        run: |
-          LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
-          gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
-          unzip -q codeql-linux64.zip
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
+        uses: ./.github/actions/fetch-codeql
       - name: Build Query Pack
         run: |
-          codeql/codeql pack create ql/lib --output target/packs
-          codeql/codeql pack install ql/src
-          codeql/codeql pack create ql/src --output target/packs
+          codeql pack create ql/lib --output target/packs
+          codeql pack install ql/src
+          codeql pack create ql/src --output target/packs
           PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
-          codeql/codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
+          codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
           (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: codeql-ruby-queries
           path: |
@@ -113,20 +112,20 @@ jobs:
     runs-on: ubuntu-latest
     needs: [build, compile-queries]
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
         with:
           name: ruby.dbscheme
           path: ruby/ruby
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
         with:
           name: extractor-ubuntu-latest
           path: ruby/linux64
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
         with:
           name: extractor-windows-latest
           path: ruby/win64
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
         with:
           name: extractor-macos-latest
           path: ruby/osx64
@@ -142,12 +141,12 @@ jobs:
           cp win64/ruby-extractor.exe ruby/tools/win64/extractor.exe
           chmod +x ruby/tools/{linux64,osx64}/{autobuilder,extractor}
           zip -rq codeql-ruby.zip ruby
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: codeql-ruby-pack
           path: ruby/codeql-ruby.zip
           retention-days: 1
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
         with:
           name: codeql-ruby-queries
           path: ruby/qlpacks
@@ -159,7 +158,7 @@ jobs:
             ]
           }' > .codeqlmanifest.json
           zip -rq codeql-ruby-bundle.zip .codeqlmanifest.json ruby qlpacks
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: codeql-ruby-bundle
           path: ruby/codeql-ruby-bundle.zip
@@ -177,21 +176,17 @@ jobs:
     runs-on: ${{ matrix.os }}
     needs: [package]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+      - name: Fetch CodeQL
+        uses: ./.github/actions/fetch-codeql
+
+      - uses: actions/checkout@v3
         with:
           repository: Shopify/example-ruby-app
           ref: 67a0decc5eb550f3a9228eda53925c3afd40dfe9
-      - name: Fetch CodeQL
-        shell: bash
-        run: |
-          LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
-          gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql.zip "$LATEST"
-          unzip -q codeql.zip
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        working-directory: ${{ runner.temp }}
+
       - name: Download Ruby bundle
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: codeql-ruby-bundle
           path: ${{ runner.temp }}
@@ -213,12 +208,12 @@ jobs:
       - name: Run QL test
         shell: bash
         run: |
-          "${{ runner.temp }}/codeql/codeql" test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
+          codeql test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
       - name: Create database
         shell: bash
         run: |
-          "${{ runner.temp }}/codeql/codeql" database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
+          codeql database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
       - name: Analyze database
         shell: bash
         run: |
-          "${{ runner.temp }}/codeql/codeql" database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls
+          codeql database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/ruby-dataset-measure.yml

@@ -27,14 +27,14 @@ jobs:
         repo: [rails/rails, discourse/discourse, spree/spree, ruby/ruby]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - uses: ./.github/actions/fetch-codeql
 
       - uses: ./ruby/actions/create-extractor-pack
 
       - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: ${{ matrix.repo }}
           path: ${{ github.workspace }}/repo
@@ -49,7 +49,7 @@ jobs:
         run: |
           mkdir -p "stats/${{ matrix.repo }}"
           codeql dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ruby"
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: measurements
           path: stats
@@ -59,15 +59,15 @@ jobs:
     runs-on: ubuntu-latest
     needs: measure
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
         with:
           name: measurements
           path: stats
       - run: |
           python -m pip install --user lxml
           find stats -name 'stats.xml' | sort | xargs python ruby/scripts/merge_stats.py --output ruby/ql/lib/ruby.dbscheme.stats --normalise ruby_tokeninfo
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: ruby.dbscheme.stats
           path: ruby/ql/lib/ruby.dbscheme.stats

.github/workflows/ruby-qltest.yml

@@ -5,6 +5,8 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-qltest.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
@@ -12,6 +14,8 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-qltest.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
@@ -27,14 +31,14 @@ jobs:
   qlformat:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - name: Check QL formatting
         run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
   qlcompile:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - name: Check QL compilation
         run: |
@@ -44,7 +48,7 @@ jobs:
   qlupgrade:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - name: Check DB upgrade scripts
         run: |
@@ -67,7 +71,7 @@ jobs:
       matrix:
         slice: ["1/2", "2/2"]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - uses: ./ruby/actions/create-extractor-pack
       - name: Run QL tests

.github/workflows/swift-codegen.yml

@@ -0,0 +1,37 @@
+name: "Swift: Check code generation"
+
+on:
+  pull_request:
+    paths:
+      - "swift/**"
+      - .github/workflows/swift-codegen.yml
+      - .github/actions/fetch-codeql/action.yml
+    branches:
+      - main
+
+jobs:
+  codegen:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v3
+      - uses: pre-commit/action@v3.0.0
+        name: Check that python code is properly formatted
+        with:
+          extra_args: autopep8 --all-files
+      - name: Run unit tests
+        run: |
+          bazel test //swift/codegen/test --test_output=errors
+      - uses: pre-commit/action@v3.0.0
+        name: Check that QL generated code was checked in
+        with:
+          extra_args: swift-codegen --all-files
+      - name: Generate C++ files
+        run: |
+          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-cpp-files
+      - uses: actions/upload-artifact@v3
+        with:
+          name: swift-generated-cpp-files
+          path: swift-generated-cpp-files/**

.github/workflows/swift-integration-tests.yml

@@ -0,0 +1,35 @@
+name: "Swift: Run Integration Tests"
+
+on:
+  pull_request:
+    paths:
+      - "swift/**"
+      - .github/workflows/swift-integration-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+defaults:
+  run:
+    working-directory: swift
+
+jobs:
+  integration-tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-20.04
+#          - macos-latest  TODO
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v3
+      - name: Build Swift extractor
+        run: |
+          bazel run //swift:create-extractor-pack
+      - name: Run integration tests
+        run: |
+          python integration-tests/runner.py

.github/workflows/swift-qltest.yml

@@ -0,0 +1,41 @@
+name: "Swift: Run QL Tests"
+
+on:
+  pull_request:
+    paths:
+      - "swift/**"
+      - .github/workflows/swift-qltest.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+defaults:
+  run:
+    working-directory: swift
+
+jobs:
+  qlformat:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - name: Check QL formatting
+        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
+  qltest:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os : [ubuntu-20.04, macos-latest]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - uses: bazelbuild/setup-bazelisk@v2
+      - name: Build Swift extractor
+        run: |
+          bazel run //swift:create-extractor-pack
+      - name: Run QL tests
+        run: |
+          codeql test run --threads=0 --ram 5000 --search-path "${{ github.workspace }}/swift/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition ql/test
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/sync-files.yml

@@ -14,7 +14,7 @@ jobs:
   sync:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Check synchronized files
         run: python config/sync-files.py
 

.github/workflows/validate-change-notes.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "*/ql/*/change-notes/**/*"
       - ".github/workflows/validate-change-notes.yml"
+      - ".github/actions/fetch-codeql/action.yml"
     branches:
       - main
       - "rc/*"
@@ -12,13 +13,14 @@ on:
     paths:
       - "*/ql/*/change-notes/**/*"
       - ".github/workflows/validate-change-notes.yml"
+      - ".github/actions/fetch-codeql/action.yml"
 
 jobs:
   check-change-note:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql

.gitignore

@@ -9,6 +9,7 @@
 # qltest projects and artifacts
 */ql/test/**/*.testproj
 */ql/test/**/*.actual
+*/ql/test/**/go.sum
 
 # Visual studio temporaries, except a file used by QL4VS
 .vs/*
@@ -17,9 +18,12 @@
 # Byte-compiled python files
 *.pyc
 
-# python virtual environment folder 
+# python virtual environment folder
 .venv/
 
+# binary files created by pytest-cov
+.coverage
+
 # It's useful (though not required) to be able to unpack codeql in the ql checkout itself
 /codeql/
 
@@ -29,4 +33,31 @@ csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
 .codeql
 
 # Compiled class file
-*.class
+*.class
+
+# links created by bazel
+/bazel-*
+
+# local bazel options
+/local.bazelrc
+
+# CLion project files
+/.clwb
+
+# Go build artifacts
+go/build/*
+
+# Go binaries
+go/tools/bin
+go/tools/linux64
+go/tools/osx64
+go/tools/win64
+go/tools/tokenizer.jar
+go/main
+
+# node_modules folders except in the JS test suite
+node_modules/
+!/javascript/ql/test/**/node_modules/
+
+# Temporary folders for working with generated models
+.model-temp

.lgtm.yml

@@ -6,6 +6,7 @@ path_classifiers:
   test:
   - csharp/ql/src
   - csharp/ql/test
+  - go/ql/test
   - javascript/extractor/parser-tests
   - javascript/extractor/tests
   - javascript/ql/src
@@ -13,6 +14,9 @@ path_classifiers:
   - python/ql/src
   - python/ql/test
 
+  example:
+  - go/ql/src
+
 queries:
   - include: "*"
 

.pre-commit-config.yaml

@@ -1,29 +1,57 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
-exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.2.0
     hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
+      - id: trailing-whitespace
+        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
+      - id: end-of-file-fixer
+        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
 
--   repo: local
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: v13.0.1
     hooks:
-    -   id: codeql-format
+      - id: clang-format
+        files: ^swift/.*\.(h|c|cpp)$
+
+  - repo: https://github.com/pre-commit/mirrors-autopep8
+    rev: v1.6.0
+    hooks:
+      - id: autopep8
+        files: ^swift/codegen/.*\.py
+
+  - repo: local
+    hooks:
+      - id: codeql-format
         name: Fix QL file formatting
         files: \.qll?$
         language: system
         entry: codeql query format --in-place
 
-    -   id: sync-files
+      - id: sync-files
         name: Fix files required to be identical
+        files: \.(qll?|qhelp|swift)$
         language: system
         entry: python3 config/sync-files.py --latest
         pass_filenames: false
 
-    -   id: qhelp
+      - id: qhelp
         name: Check query help generation
         files: \.qhelp$
         language: system
         entry: python3 misc/scripts/check-qhelp.py
+
+      - id: swift-codegen
+        name: Run Swift checked in code generation
+        files: ^swift/(codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements))
+        language: system
+        entry: bazel run //swift/codegen -- --quiet
+        pass_filenames: false
+
+      - id: swift-codegen-unit-tests
+        name: Run Swift code generation unit tests
+        files: ^swift/codegen/.*\.py$
+        language: system
+        entry: bazel test //swift/codegen/test
+        pass_filenames: false

CODEOWNERS

@@ -1,17 +1,13 @@
 /cpp/ @github/codeql-c-analysis
 /csharp/ @github/codeql-csharp
+/go/ @github/codeql-go
 /java/ @github/codeql-java
 /javascript/ @github/codeql-javascript
 /python/ @github/codeql-python
 /ruby/ @github/codeql-ruby
-
-# Make @xcorail (GitHub Security Lab) a code owner for experimental queries so he gets pinged when we promote a query out of experimental
-/cpp/**/experimental/**/* @github/codeql-c-analysis @xcorail
-/csharp/**/experimental/**/* @github/codeql-csharp @xcorail
-/java/**/experimental/**/* @github/codeql-java @xcorail
-/javascript/**/experimental/**/* @github/codeql-javascript @xcorail
-/python/**/experimental/**/* @github/codeql-python @xcorail
-/ruby/**/experimental/**/* @github/codeql-ruby @xcorail
+/swift/ @github/codeql-c
+/java/kotlin-extractor/ @github/codeql-kotlin
+/java/kotlin-explorer/ @github/codeql-kotlin
 
 # ML-powered queries
 /javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
@@ -31,3 +27,19 @@
 
 # QL for QL reviewers
 /ql/ @github/codeql-ql-for-ql-reviewers
+
+# Bazel (excluding BUILD.bazel files)
+WORKSPACE.bazel @github/codeql-ci-reviewers
+**/*.bzl @github/codeql-ci-reviewers
+
+# Documentation etc
+/*.md @github/code-scanning-product
+/LICENSE @github/code-scanning-product
+
+# Workflows
+/.github/workflows/ @github/codeql-ci-reviewers
+/.github/workflows/go-* @github/codeql-go
+/.github/workflows/js-ml-tests.yml @github/codeql-ml-powered-queries-reviewers
+/.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
+/.github/workflows/ruby-* @github/codeql-ruby
+/.github/workflows/swift-* @github/codeql-c

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 We welcome contributions to our CodeQL libraries and queries. Got an idea for a new check, or how to improve an existing query? Then please go ahead and open a pull request! Contributions to this project are [released](https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license) to the public under the [project's open source license](LICENSE).
 
-There is lots of useful documentation to help you write queries, ranging from information about query file structure to tutorials for specific target languages. For more information on the documentation available, see [CodeQL queries](https://help.semmle.com/QL/learn-ql/writing-queries/writing-queries.html) on [help.semmle.com](https://help.semmle.com).
+There is lots of useful documentation to help you write queries, ranging from information about query file structure to tutorials for specific target languages. For more information on the documentation available, see [CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/codeql-queries) on [codeql.github.com](https://codeql.github.com).
 
 ## Change notes
 
@@ -36,11 +36,11 @@ If you have an idea for a query that you would like to share with other CodeQL u
 
     For details, see the [guide on query metadata](docs/query-metadata-style-guide.md).
 
-    Make sure the `select` statement is compatible with the query `@kind`. See [About CodeQL queries](https://help.semmle.com/QL/learn-ql/writing-queries/introduction-to-queries.html#select-clause) on help.semmle.com.
+    Make sure the `select` statement is compatible with the query `@kind`. See [About CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/about-codeql-queries/#select-clause) on codeql.github.com.
 
 3. **Formatting**
 
-    - The queries and libraries must be autoformatted, for example using the "Format Document" command in [CodeQL for Visual Studio Code](https://help.semmle.com/codeql/codeql-for-vscode/procedures/about-codeql-for-vscode.html).
+    - The queries and libraries must be autoformatted, for example using the "Format Document" command in [CodeQL for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-codeql-for-visual-studio-code).
 
     If you prefer, you can either:
     1. install the [pre-commit framework](https://pre-commit.com/) and install the configured hooks on this repo via `pre-commit install`, or
@@ -70,3 +70,7 @@ After the experimental query is merged, we welcome pull requests to improve it.
 If you contribute to this project, we will record your name and email address (as provided by you with your contributions) as part of the code repositories, which are public. We might also use this information to contact you in relation to your contributions, as well as in the normal course of software development. We also store records of CLA agreements signed in the past, but no longer require contributors to sign a CLA. Under GDPR legislation, we do this on the basis of our legitimate interest in creating the CodeQL product.
 
 Please do get in touch (privacy@github.com) if you have any questions about this or our data protection policies.
+
+## Bazel
+Please notice that any bazel targets and definitions in this repository are currently experimental
+and for internal use only.

README.md

@@ -1,6 +1,6 @@
 # CodeQL
 
-This open source repository contains the standard CodeQL libraries and queries that power [GitHub Advanced Security](https://github.com/features/security/code) and the other application security products that [GitHub](https://github.com/features/security/) makes available to its customers worldwide. For the queries, libraries, and extractor that power Go analysis, visit the [CodeQL for Go repository](https://github.com/github/codeql-go).
+This open source repository contains the standard CodeQL libraries and queries that power [GitHub Advanced Security](https://github.com/features/security/code) and the other application security products that [GitHub](https://github.com/features/security/) makes available to its customers worldwide.
 
 ## How do I learn CodeQL and run queries?
 
@@ -13,7 +13,9 @@ We welcome contributions to our standard library and standard checks. Do you hav
 
 ## License
 
-The code in this repository is licensed under the [MIT License](LICENSE) by [GitHub](https://github.com). The use of CodeQL on open source code is licensed under specific [Terms & Conditions](https://securitylab.github.com/tools/codeql/license/) UNLESS you have a commercial license in place. If you'd like to use CodeQL with a commercial codebase, please [contact us](https://github.com/enterprise/contact) for further help.
+The code in this repository is licensed under the [MIT License](LICENSE) by [GitHub](https://github.com).
+
+The CodeQL CLI (including the CodeQL engine) is hosted in a [different repository](https://github.com/github/codeql-cli-binaries) and is [licensed separately](https://github.com/github/codeql-cli-binaries/blob/main/LICENSE.md). If you'd like to use the CodeQL CLI to analyze closed-source code, you will need a separate commercial license; please [contact us](https://github.com/enterprise/contact) for further help.
 
 ## Visual Studio Code integration
 

WORKSPACE.bazel

@@ -0,0 +1,12 @@
+# Please notice that any bazel targets and definitions in this repository are currently experimental
+# and for internal use only.
+
+workspace(name = "codeql")
+
+load("//misc/bazel:workspace.bzl", "codeql_workspace")
+
+codeql_workspace()
+
+load("//misc/bazel:workspace_deps.bzl", "codeql_workspace_deps")
+
+codeql_workspace_deps()

codeql-workspace.yml

@@ -0,0 +1,32 @@
+provide:
+  - "*/ql/src/qlpack.yml"
+  - "*/ql/lib/qlpack.yml"
+  - "*/ql/test/qlpack.yml"
+  - "*/ql/examples/qlpack.yml"
+  - "*/ql/consistency-queries/qlpack.yml"
+  - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
+  - "go/ql/config/legacy-support/qlpack.yml"
+  - "go/build/codeql-extractor-go/codeql-extractor.yml"
+  - "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml"
+  # This pack is explicitly excluded from the workspace since most users
+  # will want to use a version of this pack from the package cache. Internal
+  # users can uncomment the following line and place a custom ML model
+  # in the corresponding pack to test a custom ML model within their local
+  # checkout.
+  # - "javascript/ql/experimental/adaptivethreatmodeling/model/qlpack.yml"
+  - "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml"
+  - "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml"
+  - "csharp/ql/campaigns/Solorigate/lib/qlpack.yml"
+  - "csharp/ql/campaigns/Solorigate/src/qlpack.yml"
+  - "csharp/ql/campaigns/Solorigate/test/qlpack.yml"
+  - "misc/legacy-support/*/qlpack.yml"
+  - "misc/suite-helpers/qlpack.yml"
+  - "ruby/extractor-pack/codeql-extractor.yml"
+  - "swift/extractor-pack/codeql-extractor.yml"
+  - "ql/extractor-pack/codeql-extractor.ym"
+
+versionPolicies:
+  default:
+    requireChangeNotes: true
+    committedPrereleaseSuffix: dev
+    committedVersion: nextPatchRelease

config/identical-files.json

@@ -22,13 +22,15 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplForContentDataFlow.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForLibraries.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForLibraries.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"
   ],
   "DataFlow Java/C++/C#/Python Common": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll",
@@ -36,7 +38,8 @@
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll"
   ],
   "TaintTracking::Configuration Java/C++/C#/Python": [
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
@@ -51,12 +54,14 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking5/TaintTrackingImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
+    "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking1/TaintTrackingImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking2/TaintTrackingImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking3/TaintTrackingImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking4/TaintTrackingImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttrackingforlibraries/TaintTrackingImpl.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttrackingforlibraries/TaintTrackingImpl.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
   ],
   "DataFlow Java/C++/C#/Python Consistency checks": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll",
@@ -64,24 +69,22 @@
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplConsistency.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplConsistency.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplConsistency.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplConsistency.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplConsistency.qll"
   ],
   "DataFlow Java/C# Flow Summaries": [
     "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/FlowSummaryImpl.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/FlowSummaryImpl.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/FlowSummaryImpl.qll"
   ],
   "SsaReadPosition Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll"
   ],
-  "Model as Data Generation Java/C# - Utils": [
-    "java/ql/src/utils/model-generator/ModelGeneratorUtils.qll",
-    "csharp/ql/src/utils/model-generator/ModelGeneratorUtils.qll"
-  ],
-  "Model as Data Generation Java/C# - SummaryModels": [
-    "java/ql/src/utils/model-generator/CaptureSummaryModels.qll",
-    "csharp/ql/src/utils/model-generator/CaptureSummaryModels.qll"
+  "Model as Data Generation Java/C# - CaptureModels": [
+    "java/ql/src/utils/model-generator/internal/CaptureModels.qll",
+    "csharp/ql/src/utils/model-generator/internal/CaptureModels.qll"
   ],
   "Sign Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/Sign.qll",
@@ -387,7 +390,10 @@
     "csharp/ql/test/TestUtilities/InlineExpectationsTest.qll",
     "java/ql/test/TestUtilities/InlineExpectationsTest.qll",
     "python/ql/test/TestUtilities/InlineExpectationsTest.qll",
-    "ruby/ql/test/TestUtilities/InlineExpectationsTest.qll"
+    "ruby/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "ql/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "go/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "swift/ql/test/TestUtilities/InlineExpectationsTest.qll"
   ],
   "C++ ExternalAPIs": [
     "cpp/ql/src/Security/CWE/CWE-020/ExternalAPIs.qll",
@@ -448,11 +454,11 @@
     "python/ql/src/Lexical/CommentedOutCodeReferences.inc.qhelp"
   ],
   "IDE Contextual Queries": [
-    "cpp/ql/src/IDEContextual.qll",
-    "csharp/ql/src/IDEContextual.qll",
-    "java/ql/src/IDEContextual.qll",
-    "javascript/ql/src/IDEContextual.qll",
-    "python/ql/src/analysis/IDEContextual.qll"
+    "cpp/ql/lib/IDEContextual.qll",
+    "csharp/ql/lib/IDEContextual.qll",
+    "java/ql/lib/IDEContextual.qll",
+    "javascript/ql/lib/IDEContextual.qll",
+    "python/ql/lib/analysis/IDEContextual.qll"
   ],
   "SSA C#": [
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/SsaImplCommon.qll",
@@ -460,7 +466,8 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/basessa/SsaImplCommon.qll",
     "csharp/ql/lib/semmle/code/cil/internal/SsaImplCommon.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/SsaImplCommon.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaImplCommon.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaImplCommon.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/SsaImplCommon.qll"
   ],
   "CryptoAlgorithms Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/CryptoAlgorithms.qll",
@@ -477,28 +484,44 @@
     "python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll",
     "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll"
   ],
-  "ReDoS Util Python/JS/Ruby": [
-    "javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll",
-    "python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll",
-    "ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll"
+  "ReDoS Util Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/NfaUtils.qll",
+    "python/ql/lib/semmle/python/security/regexp/NfaUtils.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/NfaUtils.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/NfaUtils.qll"
   ],
-  "ReDoS Exponential Python/JS": [
-    "javascript/ql/lib/semmle/javascript/security/performance/ExponentialBackTracking.qll",
-    "python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll"
+  "ReDoS Exponential Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/ExponentialBackTracking.qll",
+    "python/ql/lib/semmle/python/security/regexp/ExponentialBackTracking.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/ExponentialBackTracking.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/ExponentialBackTracking.qll"
   ],
-  "ReDoS Polynomial Python/JS": [
-    "javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll",
-    "python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll",
-    "ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll"
+  "ReDoS Polynomial Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/SuperlinearBackTracking.qll",
+    "python/ql/lib/semmle/python/security/regexp/SuperlinearBackTracking.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/SuperlinearBackTracking.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/SuperlinearBackTracking.qll"
+  ],
+  "RegexpMatching Python/JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/regexp/RegexpMatching.qll",
+    "python/ql/lib/semmle/python/security/regexp/RegexpMatching.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/RegexpMatching.qll"
   ],
   "BadTagFilterQuery Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll",
     "python/ql/lib/semmle/python/security/BadTagFilterQuery.qll",
     "ruby/ql/lib/codeql/ruby/security/BadTagFilterQuery.qll"
   ],
+  "OverlyLargeRange Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/OverlyLargeRangeQuery.qll",
+    "python/ql/lib/semmle/python/security/OverlyLargeRangeQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/OverlyLargeRangeQuery.qll",
+    "java/ql/lib/semmle/code/java/security/OverlyLargeRangeQuery.qll"
+  ],
   "CFG": [
     "csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll",
-    "ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll"
+    "ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll",
+    "swift/ql/lib/codeql/swift/controlflow/internal/ControlFlowGraphImplShared.qll"
   ],
   "TypeTracker": [
     "python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll",
@@ -516,14 +539,67 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/AccessPathSyntax.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/AccessPathSyntax.qll",
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/AccessPathSyntax.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll",
+    "python/ql/lib/semmle/python/frameworks/data/internal/AccessPathSyntax.qll",
+    "swift/ql/lib/codeql/swift/dataflow/internal/AccessPathSyntax.qll"
+  ],
+  "IncompleteUrlSubstringSanitization": [
+    "javascript/ql/src/Security/CWE-020/IncompleteUrlSubstringSanitization.qll",
+    "ruby/ql/src/queries/security/cwe-020/IncompleteUrlSubstringSanitization.qll"
+  ],
+  "Concepts Python/Ruby/JS": [
+    "python/ql/lib/semmle/python/internal/ConceptsShared.qll",
+    "ruby/ql/lib/codeql/ruby/internal/ConceptsShared.qll",
+    "javascript/ql/lib/semmle/javascript/internal/ConceptsShared.qll"
   ],
   "Hostname Regexp queries": [
     "javascript/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
+    "python/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
     "ruby/ql/src/queries/security/cwe-020/HostnameRegexpShared.qll"
   ],
   "ApiGraphModels": [
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll",
-    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll"
+    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll",
+    "python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll"
+  ],
+  "TaintedFormatStringQuery Ruby/JS": [
+    "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/TaintedFormatStringQuery.qll"
+  ],
+  "TaintedFormatStringCustomizations Ruby/JS": [
+    "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringCustomizations.qll",
+    "ruby/ql/lib/codeql/ruby/security/TaintedFormatStringCustomizations.qll"
+  ],
+  "HttpToFileAccessQuery JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/dataflow/HttpToFileAccessQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/HttpToFileAccessQuery.qll"
+  ],
+  "HttpToFileAccessCustomizations JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/dataflow/HttpToFileAccessCustomizations.qll",
+    "ruby/ql/lib/codeql/ruby/security/HttpToFileAccessCustomizations.qll"
+  ],
+  "Typo database": [
+    "javascript/ql/src/Expressions/TypoDatabase.qll",
+    "ql/ql/src/codeql_ql/style/TypoDatabase.qll"
+  ],
+  "Swift declarations test file": [
+    "swift/ql/test/extractor-tests/declarations/declarations.swift",
+    "swift/ql/test/library-tests/parent/declarations.swift"
+  ],
+  "Swift statements test file": [
+    "swift/ql/test/extractor-tests/statements/statements.swift",
+    "swift/ql/test/library-tests/parent/statements.swift"
+  ],
+  "Swift expressions test file": [
+    "swift/ql/test/extractor-tests/expressions/expressions.swift",
+    "swift/ql/test/library-tests/parent/expressions.swift"
+  ],
+  "Swift patterns test file": [
+    "swift/ql/test/extractor-tests/patterns/patterns.swift",
+    "swift/ql/test/library-tests/parent/patterns.swift"
+  ],
+  "IncompleteMultiCharacterSanitization JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/IncompleteMultiCharacterSanitizationQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/IncompleteMultiCharacterSanitizationQuery.qll"
   ]
 }

conftest.py

@@ -0,0 +1 @@
+# this empty file adds the repo root to PYTHON_PATH when running pytest

cpp/BUILD.bazel

@@ -0,0 +1,17 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@rules_pkg//:mappings.bzl", "pkg_filegroup")
+
+alias(
+    name = "dbscheme",
+    actual = "//cpp/ql/lib:dbscheme",
+)
+
+pkg_filegroup(
+    name = "db-files",
+    srcs = [
+        ":dbscheme",
+        "//cpp/downgrades",
+        "//cpp/ql/lib:dbscheme-stats",
+    ],
+)

cpp/downgrades/19e31bf071f588bb7efd1e4d5a185ce4f6fbbd84/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add relation for tracking C++ braced initializers
+compatibility: full
+braced_initialisers.rel: delete

cpp/downgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/exprs.ql

@@ -0,0 +1,17 @@
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Location extends @location_expr {
+  string toString() { none() }
+}
+
+predicate isExprWithNewBuiltin(Expr expr) {
+  exists(int kind | exprs(expr, kind, _) | 330 <= kind and kind <= 334)
+}
+
+from Expr expr, int kind, int kind_new, Location location
+where
+  exprs(expr, kind, location) and
+  if isExprWithNewBuiltin(expr) then kind_new = 0 else kind_new = kind
+select expr, kind_new, location

cpp/downgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add new builtin operations
+compatibility: partial
+exprs.rel: run exprs.qlo

cpp/downgrades/34549c3b0937002f11037d01822ebe99442c1402/attribute_args.ql

@@ -0,0 +1,17 @@
+class AttributeArgument extends @attribute_arg {
+  string toString() { none() }
+}
+
+class Attribute extends @attribute {
+  string toString() { none() }
+}
+
+class LocationDefault extends @location_default {
+  string toString() { none() }
+}
+
+from AttributeArgument arg, int kind, Attribute attr, int index, LocationDefault location
+where
+  attribute_args(arg, kind, attr, index, location) and
+  not arg instanceof @attribute_arg_constant_expr
+select arg, kind, attr, index, location

cpp/downgrades/34549c3b0937002f11037d01822ebe99442c1402/upgrade.properties

@@ -0,0 +1,4 @@
+description: Support all constant attribute arguments
+compatibility: backwards
+attribute_arg_constant.rel: delete
+attribute_args.rel: run attribute_args.qlo

cpp/downgrades/73af5058c6899dcdb05754c27ca966aeb3a68c94/exprs.ql

@@ -0,0 +1,13 @@
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Location extends @location_expr {
+  string toString() { none() }
+}
+
+from Expr expr, int kind, int kind_new, Location location
+where
+  exprs(expr, kind, location) and
+  if expr instanceof @blockassignexpr then kind_new = 0 else kind_new = kind
+select expr, kind_new, location

cpp/downgrades/73af5058c6899dcdb05754c27ca966aeb3a68c94/upgrade.properties

@@ -0,0 +1,3 @@
+description: Support block assignment
+compatibility: partial
+exprs.rel: run exprs.qlo

cpp/downgrades/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
+
+pkg_files(
+    name = "downgrades",
+    srcs = glob(
+        ["**"],
+        exclude = ["BUILD.bazel"],
+    ),
+    prefix = "cpp/downgrades",
+    strip_prefix = strip_prefix.from_pkg(),
+    visibility = ["//cpp:__pkg__"],
+)

cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/exprparents.ql

@@ -0,0 +1,21 @@
+class Element extends @element {
+  string toString() { none() }
+}
+
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Stmt extends @stmt {
+  string toString() { none() }
+}
+
+predicate isStmtWithInitializer(Stmt stmt) {
+  exists(int kind | stmts(stmt, kind, _) | kind = 2 or kind = 11 or kind = 35)
+}
+
+from Expr child, int index, int index_new, Element parent
+where
+  exprparents(child, index, parent) and
+  if isStmtWithInitializer(parent) then index_new = index - 1 else index_new = index
+select child, index_new, parent

cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/stmtparents.ql

@@ -0,0 +1,22 @@
+class Element extends @element {
+  string toString() { none() }
+}
+
+class Stmt extends @stmt {
+  string toString() { none() }
+}
+
+predicate isStmtWithInitializer(Stmt stmt) {
+  exists(int kind | stmts(stmt, kind, _) | kind = 2 or kind = 11 or kind = 35)
+}
+
+from Stmt child, int index, int index_new, Element parent
+where
+  stmtparents(child, index, parent) and
+  (
+    not isStmtWithInitializer(parent)
+    or
+    index > 0
+  ) and
+  if isStmtWithInitializer(parent) then index_new = index - 1 else index_new = index
+select child, index_new, parent

cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/upgrade.properties

@@ -0,0 +1,6 @@
+description: Support C++17 if and switch initializers
+compatibility: partial
+if_initialization.rel: delete
+switch_initialization.rel: delete
+exprparents.rel: run exprparents.qlo
+stmtparents.rel: run stmtparents.qlo

cpp/downgrades/f96ad9b2da43bbc9e55a72a165febd270ae07981/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add relation for orphaned local variables
+compatibility: full
+orphaned_variables.rel: delete

cpp/ql/lib/BUILD.bazel

@@ -0,0 +1,15 @@
+package(default_visibility = ["//cpp:__pkg__"])
+
+load("@rules_pkg//:mappings.bzl", "pkg_files")
+
+pkg_files(
+    name = "dbscheme",
+    srcs = ["semmlecode.cpp.dbscheme"],
+    prefix = "cpp",
+)
+
+pkg_files(
+    name = "dbscheme-stats",
+    srcs = ["semmlecode.cpp.dbscheme.stats"],
+    prefix = "cpp",
+)

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,96 @@
+## 0.3.4
+
+### Deprecated APIs
+
+* Many classes/predicates/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.
+
+### New Features
+
+* Added support for getting the link targets of global and namespace variables.
+* Added a `BlockAssignExpr` class, which models a `memcpy`-like operation used in compiler generated copy/move constructors and assignment operations.
+
+### Minor Analysis Improvements
+
+* All deprecated predicates/classes/modules that have been deprecated for over a year have been deleted.
+
+## 0.3.3
+
+### New Features
+
+* Added a predicate `getValueConstant` to `AttributeArgument` that yields the argument value as an `Expr` when the value is a constant expression.
+* A new class predicate `MustFlowConfiguration::allowInterproceduralFlow` has been added to the `semmle.code.cpp.ir.dataflow.MustFlow` library. The new predicate can be overridden to disable interprocedural flow.
+* Added subclasses of `BuiltInOperations` for `__builtin_bit_cast`, `__builtin_shuffle`, `__has_unique_object_representations`, `__is_aggregate`, and `__is_assignable`.
+
+### Major Analysis Improvements
+
+* The IR dataflow library now includes flow through global variables. This enables new findings in many scenarios.
+
+## 0.3.2
+
+### Bug Fixes
+
+* Under certain circumstances a variable declaration that is not also a definition could be associated with a `Variable` that did not have the definition as a `VariableDeclarationEntry`. This is now fixed, and a unique `Variable` will exist that has both the declaration and the definition as a `VariableDeclarationEntry`.
+
+## 0.3.1
+
+### Minor Analysis Improvements
+
+* `AnalysedExpr::isNullCheck` and `AnalysedExpr::isValidCheck` have been updated to handle variable accesses on the left-hand side of the C++ logical "and", and variable declarations in conditions.
+
+## 0.3.0
+
+### Deprecated APIs
+
+* The `BarrierGuard` class has been deprecated. Such barriers and sanitizers can now instead be created using the new `BarrierGuard` parameterized module.
+
+### Bug Fixes
+
+* `UserType.getADeclarationEntry()` now yields all forward declarations when the user type is a `class`, `struct`, or `union`.
+
+## 0.2.3
+
+### New Features
+
+* An `isBraced` predicate was added to the `Initializer` class which holds when a C++ braced initializer was used in the initialization.
+
+## 0.2.2
+
+### Deprecated APIs
+
+ * The `AnalysedString` class in the `StringAnalysis` module has been replaced with `AnalyzedString`, to follow our style guide. The old name still exists as a deprecated alias.
+
+### New Features
+
+* A `getInitialization` predicate was added to the `ConstexprIfStmt`, `IfStmt`, and `SwitchStmt` classes that yields the C++17-style initializer of the `if` or `switch` statement when it exists.
+
+## 0.2.1
+
+## 0.2.0
+
+### Breaking Changes
+
+* The signature of `allowImplicitRead` on `DataFlow::Configuration` and `TaintTracking::Configuration` has changed from `allowImplicitRead(DataFlow::Node node, DataFlow::Content c)` to `allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c)`.
+
+### Minor Analysis Improvements
+
+* More Windows pool allocation functions are now detected as `AllocationFunction`s.
+* The `semmle.code.cpp.commons.Buffer` library has been enhanced to handle array members of classes that do not specify a size.
+
+## 0.1.0
+
+### Breaking Changes
+
+* The recently added flow-state versions of `isBarrierIn`, `isBarrierOut`, `isSanitizerIn`, and `isSanitizerOut` in the data flow and taint tracking libraries have been removed.
+
+### New Features
+
+* A new library `semmle.code.cpp.security.PrivateData` has been added. The new library heuristically detects variables and functions dealing with sensitive private data, such as e-mail addresses and credit card numbers.
+
+### Minor Analysis Improvements
+
+* The `semmle.code.cpp.security.SensitiveExprs` library has been enhanced with some additional rules for detecting credentials.
+
 ## 0.0.13
 
 ## 0.0.12

cpp/ql/lib/change-notes/released/0.1.0.md

@@ -0,0 +1,13 @@
+## 0.1.0
+
+### Breaking Changes
+
+* The recently added flow-state versions of `isBarrierIn`, `isBarrierOut`, `isSanitizerIn`, and `isSanitizerOut` in the data flow and taint tracking libraries have been removed.
+
+### New Features
+
+* A new library `semmle.code.cpp.security.PrivateData` has been added. The new library heuristically detects variables and functions dealing with sensitive private data, such as e-mail addresses and credit card numbers.
+
+### Minor Analysis Improvements
+
+* The `semmle.code.cpp.security.SensitiveExprs` library has been enhanced with some additional rules for detecting credentials.

cpp/ql/lib/change-notes/released/0.2.0.md

@@ -0,0 +1,10 @@
+## 0.2.0
+
+### Breaking Changes
+
+* The signature of `allowImplicitRead` on `DataFlow::Configuration` and `TaintTracking::Configuration` has changed from `allowImplicitRead(DataFlow::Node node, DataFlow::Content c)` to `allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c)`.
+
+### Minor Analysis Improvements
+
+* More Windows pool allocation functions are now detected as `AllocationFunction`s.
+* The `semmle.code.cpp.commons.Buffer` library has been enhanced to handle array members of classes that do not specify a size.

cpp/ql/lib/change-notes/released/0.2.1.md

@@ -0,0 +1 @@
+## 0.2.1

cpp/ql/lib/change-notes/released/0.2.2.md

@@ -0,0 +1,9 @@
+## 0.2.2
+
+### Deprecated APIs
+
+ * The `AnalysedString` class in the `StringAnalysis` module has been replaced with `AnalyzedString`, to follow our style guide. The old name still exists as a deprecated alias.
+
+### New Features
+
+* A `getInitialization` predicate was added to the `ConstexprIfStmt`, `IfStmt`, and `SwitchStmt` classes that yields the C++17-style initializer of the `if` or `switch` statement when it exists.

cpp/ql/lib/change-notes/released/0.2.3.md

@@ -0,0 +1,5 @@
+## 0.2.3
+
+### New Features
+
+* An `isBraced` predicate was added to the `Initializer` class which holds when a C++ braced initializer was used in the initialization.

cpp/ql/lib/change-notes/released/0.3.0.md

@@ -0,0 +1,9 @@
+## 0.3.0
+
+### Deprecated APIs
+
+* The `BarrierGuard` class has been deprecated. Such barriers and sanitizers can now instead be created using the new `BarrierGuard` parameterized module.
+
+### Bug Fixes
+
+* `UserType.getADeclarationEntry()` now yields all forward declarations when the user type is a `class`, `struct`, or `union`.

cpp/ql/lib/change-notes/released/0.3.1.md

@@ -0,0 +1,5 @@
+## 0.3.1
+
+### Minor Analysis Improvements
+
+* `AnalysedExpr::isNullCheck` and `AnalysedExpr::isValidCheck` have been updated to handle variable accesses on the left-hand side of the C++ logical "and", and variable declarations in conditions.

cpp/ql/lib/change-notes/released/0.3.2.md

@@ -0,0 +1,5 @@
+## 0.3.2
+
+### Bug Fixes
+
+* Under certain circumstances a variable declaration that is not also a definition could be associated with a `Variable` that did not have the definition as a `VariableDeclarationEntry`. This is now fixed, and a unique `Variable` will exist that has both the declaration and the definition as a `VariableDeclarationEntry`.

cpp/ql/lib/change-notes/released/0.3.3.md

@@ -0,0 +1,11 @@
+## 0.3.3
+
+### New Features
+
+* Added a predicate `getValueConstant` to `AttributeArgument` that yields the argument value as an `Expr` when the value is a constant expression.
+* A new class predicate `MustFlowConfiguration::allowInterproceduralFlow` has been added to the `semmle.code.cpp.ir.dataflow.MustFlow` library. The new predicate can be overridden to disable interprocedural flow.
+* Added subclasses of `BuiltInOperations` for `__builtin_bit_cast`, `__builtin_shuffle`, `__has_unique_object_representations`, `__is_aggregate`, and `__is_assignable`.
+
+### Major Analysis Improvements
+
+* The IR dataflow library now includes flow through global variables. This enables new findings in many scenarios.

cpp/ql/lib/change-notes/released/0.3.4.md

@@ -0,0 +1,15 @@
+## 0.3.4
+
+### Deprecated APIs
+
+* Many classes/predicates/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.
+
+### New Features
+
+* Added support for getting the link targets of global and namespace variables.
+* Added a `BlockAssignExpr` class, which models a `memcpy`-like operation used in compiler generated copy/move constructors and assignment operations.
+
+### Minor Analysis Improvements
+
+* All deprecated predicates/classes/modules that have been deprecated for over a year have been deleted.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.0.13
+lastReleaseVersion: 0.3.4

cpp/ql/lib/experimental/semmle/code/cpp/security/PrivateCleartextWrite.qll

@@ -4,7 +4,7 @@
 
 import cpp
 import semmle.code.cpp.dataflow.TaintTracking
-import experimental.semmle.code.cpp.security.PrivateData
+import semmle.code.cpp.security.PrivateData
 import semmle.code.cpp.security.FileWrite
 import semmle.code.cpp.security.BufferWrite
 

cpp/ql/lib/experimental/semmle/code/cpp/security/PrivateData.qll

@@ -1,52 +0,0 @@
-/**
- * Provides classes and predicates for identifying private data and functions for security.
- *
- * 'Private' data in general is anything that would compromise user privacy if exposed. This
- * library tries to guess where private data may either be stored in a variable or produced by a
- * function.
- *
- * This library is not concerned with credentials. See `SensitiveActions` for expressions related
- * to credentials.
- */
-
-import cpp
-
-/** A string for `match` that identifies strings that look like they represent private data. */
-private string privateNames() {
-  result =
-    [
-      // Inspired by the list on https://cwe.mitre.org/data/definitions/359.html
-      // Government identifiers, such as Social Security Numbers
-      "%social%security%number%",
-      // Contact information, such as home addresses and telephone numbers
-      "%postcode%", "%zipcode%",
-      // result = "%telephone%" or
-      // Geographic location - where the user is (or was)
-      "%latitude%", "%longitude%",
-      // Financial data - such as credit card numbers, salary, bank accounts, and debts
-      "%creditcard%", "%salary%", "%bankaccount%",
-      // Communications - e-mail addresses, private e-mail messages, SMS text messages, chat logs, etc.
-      // result = "%email%" or
-      // result = "%mobile%" or
-      "%employer%",
-      // Health - medical conditions, insurance status, prescription records
-      "%medical%"
-    ]
-}
-
-/** An expression that might contain private data. */
-abstract class PrivateDataExpr extends Expr { }
-
-/** A functiond call that might produce private data. */
-class PrivateFunctionCall extends PrivateDataExpr, FunctionCall {
-  PrivateFunctionCall() {
-    exists(string s | this.getTarget().getName().toLowerCase() = s | s.matches(privateNames()))
-  }
-}
-
-/** An access to a variable that might contain private data. */
-class PrivateVariableAccess extends PrivateDataExpr, VariableAccess {
-  PrivateVariableAccess() {
-    exists(string s | this.getTarget().getName().toLowerCase() = s | s.matches(privateNames()))
-  }
-}

cpp/ql/lib/experimental/semmle/code/cpp/semantic/Semantic.qll

@@ -0,0 +1,7 @@
+import SemanticExpr
+import SemanticBound
+import SemanticSSA
+import SemanticGuard
+import SemanticCFG
+import SemanticType
+import SemanticOpcode