ML: include Unknown endpoints in extraction query for training data

.codeqlmanifest.json

@@ -0,0 +1,27 @@
+{
+    "provide": [
+        "*/ql/src/qlpack.yml",
+        "*/ql/lib/qlpack.yml",
+        "*/ql/test/qlpack.yml",
+        "*/ql/examples/qlpack.yml",
+        "*/ql/consistency-queries/qlpack.yml",
+        "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
+        "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
+        "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml",
+        "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
+        "csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
+        "csharp/ql/campaigns/Solorigate/src/qlpack.yml",
+        "csharp/ql/campaigns/Solorigate/test/qlpack.yml",
+        "misc/legacy-support/*/qlpack.yml",
+        "misc/suite-helpers/qlpack.yml",
+        "ruby/extractor-pack/codeql-extractor.yml",
+        "ql/extractor-pack/codeql-extractor.yml"
+    ],
+    "versionPolicies": {
+      "default": {
+        "requireChangeNotes": true,
+        "committedPrereleaseSuffix": "dev",
+        "committedVersion": "nextPatchRelease"
+      }
+    }
+}

.devcontainer/devcontainer.json

@@ -3,8 +3,6 @@
     "rust-lang.rust",
     "bungcip.better-toml",
     "github.vscode-codeql",
-    "hbenl.vscode-test-explorer",
-    "ms-vscode.test-adapter-converter",
     "slevesque.vscode-zipexplorer"
   ],
   "settings": {

.devcontainer/swift/Dockerfile

@@ -1,9 +0,0 @@
-# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.236.0/containers/cpp/.devcontainer/base.Dockerfile
-
-# [Choice] Debian / Ubuntu version (use Debian 11, Ubuntu 18.04/22.04 on local arm64/Apple Silicon): debian-11, debian-10, ubuntu-22.04, ubuntu-20.04, ubuntu-18.04
-FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-ubuntu-22.04
-
-USER root
-ADD root.sh /tmp/root.sh
-ADD update-codeql.sh /usr/local/bin/update-codeql
-RUN bash /tmp/root.sh && rm /tmp/root.sh

.devcontainer/swift/devcontainer.json

@@ -1,25 +0,0 @@
-{
-	"extensions": [
-		"github.vscode-codeql",
-		"hbenl.vscode-test-explorer",
-		"ms-vscode.test-adapter-converter",
-		"slevesque.vscode-zipexplorer",
-		"ms-vscode.cpptools"
-	],
-	"settings": {
-		"files.watcherExclude": {
-			"**/target/**": true
-		},
-		"codeQL.runningQueries.memory": 2048
-	},
-	"build": {
-		"dockerfile": "Dockerfile",
-	},
-	"runArgs": [
-		"--cap-add=SYS_PTRACE",
-		"--security-opt",
-		"seccomp=unconfined"
-	],
-	"remoteUser": "vscode",
-	"onCreateCommand": ".devcontainer/swift/user.sh"
-}

.devcontainer/swift/root.sh

@@ -1,22 +0,0 @@
-set -xe
-
-BAZELISK_VERSION=v1.12.0
-BAZELISK_DOWNLOAD_SHA=6b0bcb2ea15bca16fffabe6fda75803440375354c085480fe361d2cbf32501db
-
-apt-get update
-export DEBIAN_FRONTEND=noninteractive
-apt-get -y install --no-install-recommends \
-    zlib1g-dev \
-    uuid-dev \
-    python3-distutils \
-    python3-pip \
-    bash-completion
-
-# Install Bazel
-curl -fSsL -o /usr/local/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-amd64
-echo "${BAZELISK_DOWNLOAD_SHA} */usr/local/bin/bazelisk" | sha256sum --check -
-chmod 0755 /usr/local/bin/bazelisk
-ln -s bazelisk /usr/local/bin/bazel
-
-# install latest codeql
-update-codeql

.devcontainer/swift/update-codeql.sh

@@ -1,20 +0,0 @@
-#!/bin/bash -e
-
-URL=https://github.com/github/codeql-cli-binaries/releases
-LATEST_VERSION=$(curl -L -s -H 'Accept: application/json' $URL/latest | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/')
-CURRENT_VERSION=v$(codeql version 2>/dev/null | sed -ne 's/.*release \([0-9.]*\)\./\1/p')
-if [[ $CURRENT_VERSION != $LATEST_VERSION ]]; then
-  if [[ $UID != 0 ]]; then
-    echo "update required, please run this script with sudo:"
-    echo "  sudo $0"
-    exit 1
-  fi
-  ZIP=$(mktemp codeql.XXXX.zip)
-  curl -fSqL -o $ZIP $URL/download/$LATEST_VERSION/codeql-linux64.zip
-  unzip -q $ZIP -d /opt
-  rm $ZIP
-  ln -sf /opt/codeql/codeql /usr/local/bin/codeql
-  echo installed version $LATEST_VERSION
-else
-  echo current version $CURRENT_VERSION is up-to-date
-fi

.devcontainer/swift/user.sh

@@ -1,13 +0,0 @@
-set -xe
-
-# add the workspace to the codeql search path
-mkdir -p /home/vscode/.config/codeql
-echo "--search-path /workspaces/codeql" > /home/vscode/.config/codeql/config
-
-# create a swift extractor pack with the current state
-cd /workspaces/codeql
-bazel run swift/create-extractor-pack
-
-#install and set up pre-commit
-python3 -m pip install pre-commit --no-warn-script-location
-$HOME/.local/bin/pre-commit install

.gitattributes

@@ -39,7 +39,6 @@
 *.py text
 *.lua text
 *.expected text
-*.go text
 
 # Explicitly set a bunch of known extensions to binary, because Git < 2.10 will treat
 # `* text=auto eol=lf` as `* text eol=lf`
@@ -53,14 +52,6 @@
 java/ql/test/stubs/**/*.java linguist-generated=true
 java/ql/test/experimental/stubs/**/*.java linguist-generated=true
 
-# Force git not to modify line endings for go or html files under the go/ql directory
-go/ql/**/*.go -text
-go/ql/**/*.html -text
-# Force git not to modify line endings for go dbschemes
-go/*.dbscheme -text
-# Preserve unusual line ending from codeql-go merge
-go/extractor/opencsv/CSVReader.java -text
-
 # For some languages, upgrade script testing references really old dbscheme
 # files from legacy upgrades that have CRLF line endings. Since upgrade
 # resolution relies on object hashes, we must suppress line ending conversion

.github/actions/fetch-codeql/action.yml

@@ -3,12 +3,22 @@ description: Fetches the latest version of CodeQL
 runs:
   using: composite
   steps:
+    - name: Select platform - Linux
+      if: runner.os == 'Linux'
+      shell: bash
+      run: echo "GA_CODEQL_CLI_PLATFORM=linux64" >> $GITHUB_ENV
+
+    - name: Select platform - MacOS
+      if: runner.os == 'MacOS'
+      shell: bash
+      run: echo "GA_CODEQL_CLI_PLATFORM=osx64" >> $GITHUB_ENV
+
     - name: Fetch CodeQL
       shell: bash
       run: |
-        gh extension install github/gh-codeql
-        gh codeql set-channel nightly
-        gh codeql version
-        gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_PATH}"
+        LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
+        gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-$GA_CODEQL_CLI_PLATFORM.zip "$LATEST"
+        unzip -q -d "${RUNNER_TEMP}" codeql-$GA_CODEQL_CLI_PLATFORM.zip
+        echo "${RUNNER_TEMP}/codeql" >> "${GITHUB_PATH}"
       env:
         GITHUB_TOKEN: ${{ github.token }}

.github/labeler.yml

@@ -6,23 +6,14 @@
   - csharp/**/*
   - change-notes/**/*csharp*
 
-Go:
-  - go/**/*
-  - change-notes/**/*go.*
-
 Java:
-  - any: [ 'java/**/*', '!java/kotlin-extractor/**/*', '!java/kotlin-explorer/**/*', '!java/ql/test/kotlin/**/*' ]
+  - java/**/*
   - change-notes/**/*java.*
 
 JS:
-  - any: [ 'javascript/**/*', '!javascript/ql/experimental/adaptivethreatmodeling/**/*' ]
+  - javascript/**/*
   - change-notes/**/*javascript*
 
-Kotlin:
-  - java/kotlin-extractor/**/*
-  - java/kotlin-explorer/**/*
-  - java/ql/test/kotlin/**/*
-
 Python:
   - python/**/*
   - change-notes/**/*python*
@@ -30,7 +21,7 @@ Python:
 Ruby:
   - ruby/**/*
   - change-notes/**/*ruby*
-
+  
 Swift:
   - swift/**/*
   - change-notes/**/*swift*
@@ -40,6 +31,5 @@ documentation:
   - "**/*.md"
   - docs/**/*
 
-"QL-for-QL":
+"QL-for-QL": 
   - ql/**/*
-  - .github/workflows/ql-for-ql*

.github/problem-matchers/codeql-query-format.json

@@ -1,14 +0,0 @@
-{
-    "problemMatcher": [
-        {
-            "owner": "codeql-query-format",
-            "pattern": [
-                {
-                    "regexp": "^((.*) would change by autoformatting\\.)$",
-                    "file": 2,
-                    "message": 1
-                }
-            ]
-        }
-    ]
-}

.github/problem-matchers/codeql-syntax-check.json

@@ -1,17 +0,0 @@
-{
-    "problemMatcher": [
-        {
-            "owner": "codeql-syntax-check",
-            "pattern": [
-                {
-                    "regexp": "^((ERROR|WARNING): .* \\((.*):(\\d+),(\\d+)-\\d+\\))$",
-                    "message": 1,
-                    "file": 3,
-                    "line": 4,
-                    "col": 5,
-                    "severity": 2
-                }
-            ]
-        }
-    ]
-}

.github/problem-matchers/codeql-test-run.json

@@ -1,14 +0,0 @@
-{
-    "problemMatcher": [
-        {
-            "owner": "codeql-test-run",
-            "pattern": [
-                {
-                    "regexp": "(\\[.*\\] FAILED\\((RESULT|COMPILATION)\\) (.*))$",
-                    "file": 3,
-                    "message": 1
-                }
-            ]
-        }
-    ]
-}

.github/problem-matchers/make.json

@@ -1,13 +0,0 @@
-{
-    "problemMatcher": [
-        {
-            "owner": "make",
-            "pattern": [
-                {
-                    "regexp": "^(make: \\*\\*\\* .*)$",
-                    "message": 1
-                }
-            ]
-        }
-    ]
-}

.github/workflows/check-change-note.yml

@@ -10,7 +10,6 @@ on:
       - "*/ql/lib/**/*.qll"
       - "!**/experimental/**"
       - "!ql/**"
-      - "!swift/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:

.github/workflows/check-qldoc.yml

@@ -5,7 +5,6 @@ on:
     paths:
       - "*/ql/lib/**"
       - .github/workflows/check-qldoc.yml
-      - .github/actions/fetch-codeql/action.yml
     branches:
       - main
       - "rc/*"
@@ -15,13 +14,18 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Install CodeQL
+        run: |
+          gh extension install github/gh-codeql
+          gh codeql set-channel nightly
+          gh codeql version
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
       - uses: actions/checkout@v3
         with:
           fetch-depth: 2
 
-      - name: Install CodeQL
-        uses: ./.github/actions/fetch-codeql
-
       - name: Check QLdoc coverage
         shell: bash
         run: |
@@ -30,7 +34,7 @@ jobs:
           changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!swift)[a-z]*/ql/lib' || true; } | sort -u)"
           for pack_dir in ${changed_lib_packs}; do
             lang="${pack_dir%/ql/lib}"
-            codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
+            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
           done
           git checkout HEAD^
           for pack_dir in ${changed_lib_packs}; do
@@ -38,7 +42,7 @@ jobs:
             # In this case the right thing to do is to skip the check.
             [[ ! -d "${pack_dir}" ]] && continue
             lang="${pack_dir%/ql/lib}"
-            codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
+            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
             awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-current.txt" | sort -u > "${RUNNER_TEMP}/current-undocumented.txt"
             awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-baseline.txt" | sort -u > "${RUNNER_TEMP}/baseline-undocumented.txt"
             UNDOCUMENTED="$(grep -f <(comm -13 "${RUNNER_TEMP}/baseline-undocumented.txt" "${RUNNER_TEMP}/current-undocumented.txt") "${RUNNER_TEMP}/${lang}-current.txt" || true)"

.github/workflows/codeql-analysis.yml

@@ -30,7 +30,7 @@ jobs:
     - name: Setup dotnet
       uses: actions/setup-dotnet@v2
       with:
-        dotnet-version: 6.0.202
+        dotnet-version: 6.0.101
 
     - name: Checkout repository
       uses: actions/checkout@v3
@@ -56,7 +56,7 @@ jobs:
     #    uses a compiled language
 
     - run: |
-       dotnet build csharp
+       dotnet build csharp /p:UseSharedCompilation=false
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@main

.github/workflows/csv-coverage-metrics.yml

@@ -12,7 +12,6 @@ on:
       - main
     paths:
       - ".github/workflows/csv-coverage-metrics.yml"
-      - ".github/actions/fetch-codeql/action.yml"
 
 jobs:
   publish-java:
@@ -39,7 +38,7 @@ jobs:
           path: metrics-java.sarif
           retention-days: 20
       - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@main
+        uses: github/codeql-action/upload-sarif@v1
         with:
           sarif_file: metrics-java.sarif
   
@@ -55,7 +54,7 @@ jobs:
           DATABASE="${{ runner.temp }}/csharp-database"
           PROJECT="${{ runner.temp }}/csharp-project"
           dotnet new classlib --language=C# --output="$PROJECT"
-          codeql database create "$DATABASE" --language=csharp --source-root="$PROJECT" --command 'dotnet build /t:rebuild csharp-project.csproj'
+          codeql database create "$DATABASE" --language=csharp --source-root="$PROJECT" --command 'dotnet build /t:rebuild csharp-project.csproj /p:UseSharedCompilation=false'
       - name: Capture coverage information
         run: |
           DATABASE="${{ runner.temp }}/csharp-database"
@@ -66,6 +65,6 @@ jobs:
           path: metrics-csharp.sarif
           retention-days: 20
       - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@main
+        uses: github/codeql-action/upload-sarif@v1
         with:
           sarif_file: metrics-csharp.sarif

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -3,20 +3,18 @@ name: Check framework coverage changes
 on:
   pull_request:
     paths:
-      - ".github/workflows/csv-coverage-pr-comment.yml"
-      - ".github/workflows/csv-coverage-pr-artifacts.yml"
-      - ".github/actions/fetch-codeql/action.yml"
-      - "*/ql/src/**/*.ql"
-      - "*/ql/src/**/*.qll"
-      - "*/ql/lib/**/*.ql"
-      - "*/ql/lib/**/*.qll"
-      - "misc/scripts/library-coverage/*.py"
+      - '.github/workflows/csv-coverage-pr-comment.yml'
+      - '*/ql/src/**/*.ql'
+      - '*/ql/src/**/*.qll'
+      - '*/ql/lib/**/*.ql'
+      - '*/ql/lib/**/*.qll'
+      - 'misc/scripts/library-coverage/*.py'
       # input data files
-      - "*/documentation/library-coverage/cwe-sink.csv"
-      - "*/documentation/library-coverage/frameworks.csv"
+      - '*/documentation/library-coverage/cwe-sink.csv'
+      - '*/documentation/library-coverage/frameworks.csv'
     branches:
       - main
-      - "rc/*"
+      - 'rc/*'
 
 jobs:
   generate:
@@ -25,72 +23,77 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Dump GitHub context
-        env:
-          GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-        run: echo "$GITHUB_CONTEXT"
-      - name: Clone self (github/codeql) - MERGE
-        uses: actions/checkout@v3
-        with:
-          path: merge
-      - name: Clone self (github/codeql) - BASE
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-          path: base
-      - run: |
-          git checkout HEAD^1
-          git log -1 --format='%H'
-        working-directory: base
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - name: Download CodeQL CLI
-        uses: ./merge/.github/actions/fetch-codeql
-      - name: Generate CSV files on merge commit of the PR
-        run: |
-          echo "Running generator on merge"
-          python merge/misc/scripts/library-coverage/generate-report.py ci merge merge
-          mkdir out_merge
-          cp framework-coverage-*.csv out_merge/
-          cp framework-coverage-*.rst out_merge/
-      - name: Generate CSV files on base commit of the PR
-        run: |
-          echo "Running generator on base"
-          python base/misc/scripts/library-coverage/generate-report.py ci base base
-          mkdir out_base
-          cp framework-coverage-*.csv out_base/
-          cp framework-coverage-*.rst out_base/
-      - name: Generate diff of coverage reports
-        run: |
-          python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
-      - name: Upload CSV package list
-        uses: actions/upload-artifact@v3
-        with:
-          name: csv-framework-coverage-merge
-          path: |
-            out_merge/framework-coverage-*.csv
-            out_merge/framework-coverage-*.rst
-      - name: Upload CSV package list
-        uses: actions/upload-artifact@v3
-        with:
-          name: csv-framework-coverage-base
-          path: |
-            out_base/framework-coverage-*.csv
-            out_base/framework-coverage-*.rst
-      - name: Upload comparison results
-        uses: actions/upload-artifact@v3
-        with:
-          name: comparison
-          path: |
-            comparison.md
-      - name: Save PR number
-        run: |
-          mkdir -p pr
-          echo ${{ github.event.pull_request.number }} > pr/NR
-      - name: Upload PR number
-        uses: actions/upload-artifact@v3
-        with:
-          name: pr
-          path: pr/
+    - name: Dump GitHub context
+      env:
+        GITHUB_CONTEXT: ${{ toJSON(github.event) }}
+      run: echo "$GITHUB_CONTEXT"
+    - name: Clone self (github/codeql) - MERGE
+      uses: actions/checkout@v3
+      with:
+        path: merge
+    - name: Clone self (github/codeql) - BASE
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 2
+        path: base
+    - run: |
+        git checkout HEAD^1
+        git log -1 --format='%H'
+      working-directory: base
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: 3.8
+    - name: Download CodeQL CLI
+      env:
+         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
+    - name: Unzip CodeQL CLI
+      run: unzip -d codeql-cli codeql-linux64.zip
+    - name: Generate CSV files on merge commit of the PR
+      run: |
+        echo "Running generator on merge"
+        PATH="$PATH:codeql-cli/codeql" python merge/misc/scripts/library-coverage/generate-report.py ci merge merge
+        mkdir out_merge
+        cp framework-coverage-*.csv out_merge/
+        cp framework-coverage-*.rst out_merge/
+    - name: Generate CSV files on base commit of the PR
+      run: |
+        echo "Running generator on base"
+        PATH="$PATH:codeql-cli/codeql" python base/misc/scripts/library-coverage/generate-report.py ci base base
+        mkdir out_base
+        cp framework-coverage-*.csv out_base/
+        cp framework-coverage-*.rst out_base/
+    - name: Generate diff of coverage reports
+      run: |
+        python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
+    - name: Upload CSV package list
+      uses: actions/upload-artifact@v3
+      with:
+        name: csv-framework-coverage-merge
+        path: |
+          out_merge/framework-coverage-*.csv
+          out_merge/framework-coverage-*.rst
+    - name: Upload CSV package list
+      uses: actions/upload-artifact@v3
+      with:
+        name: csv-framework-coverage-base
+        path: |
+          out_base/framework-coverage-*.csv
+          out_base/framework-coverage-*.rst
+    - name: Upload comparison results
+      uses: actions/upload-artifact@v3
+      with:
+        name: comparison
+        path: |
+          comparison.md
+    - name: Save PR number
+      run: |
+        mkdir -p pr
+        echo ${{ github.event.pull_request.number }} > pr/NR
+    - name: Upload PR number
+      uses: actions/upload-artifact@v3
+      with:
+        name: pr
+        path: pr/

.github/workflows/csv-coverage-pr-comment.yml

@@ -22,7 +22,7 @@ jobs:
     - name: Clone self (github/codeql)
       uses: actions/checkout@v3
     - name: Set up Python 3.8
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v3
       with:
         python-version: 3.8
 

.github/workflows/csv-coverage-timeseries.yml

@@ -5,29 +5,38 @@ on:
 
 jobs:
   build:
+
     runs-on: ubuntu-latest
 
     steps:
-      - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
-        with:
-          path: script
-      - name: Clone self (github/codeql) for analysis
-        uses: actions/checkout@v3
-        with:
-          path: codeqlModels
-          fetch-depth: 0
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - name: Download CodeQL CLI
-        uses: ./script/.github/actions/fetch-codeql
-      - name: Build modeled package list
-        run: |
-          python script/misc/scripts/library-coverage/generate-timeseries.py codeqlModels
-      - name: Upload timeseries CSV
-        uses: actions/upload-artifact@v3
-        with:
-          name: framework-coverage-timeseries
-          path: framework-coverage-timeseries-*.csv
+    - name: Clone self (github/codeql)
+      uses: actions/checkout@v3
+      with:
+        path: script
+    - name: Clone self (github/codeql) for analysis
+      uses: actions/checkout@v3
+      with:
+        path: codeqlModels
+        fetch-depth: 0
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: 3.8
+    - name: Download CodeQL CLI
+      env:
+         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
+    - name: Unzip CodeQL CLI
+      run: unzip -d codeql-cli codeql-linux64.zip
+    - name: Build modeled package list
+      run: |
+        CLI=$(realpath "codeql-cli/codeql")
+        echo $CLI
+        PATH="$PATH:$CLI" python script/misc/scripts/library-coverage/generate-timeseries.py codeqlModels
+    - name: Upload timeseries CSV
+      uses: actions/upload-artifact@v3
+      with:
+        name: framework-coverage-timeseries
+        path: framework-coverage-timeseries-*.csv
+

.github/workflows/csv-coverage-update.yml

@@ -12,27 +12,33 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Dump GitHub context
-        env:
-          GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-        run: echo "$GITHUB_CONTEXT"
-      - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
-        with:
-          path: ql
-          fetch-depth: 0
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - name: Download CodeQL CLI
-        uses: ./ql/.github/actions/fetch-codeql
-      - name: Generate coverage files
-        run: |
-          python ql/misc/scripts/library-coverage/generate-report.py ci ql ql
+    - name: Dump GitHub context
+      env:
+        GITHUB_CONTEXT: ${{ toJSON(github.event) }}
+      run: echo "$GITHUB_CONTEXT"
+    - name: Clone self (github/codeql)
+      uses: actions/checkout@v3
+      with:
+        path: ql
+        fetch-depth: 0
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: 3.8
+    - name: Download CodeQL CLI
+      env:
+         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
+    - name: Unzip CodeQL CLI
+      run: unzip -d codeql-cli codeql-linux64.zip
 
-      - name: Create pull request with changes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          python ql/misc/scripts/library-coverage/create-pr.py ql "$GITHUB_REPOSITORY"
+    - name: Generate coverage files
+      run: |
+        PATH="$PATH:codeql-cli/codeql" python ql/misc/scripts/library-coverage/generate-report.py ci ql ql
+
+    - name: Create pull request with changes
+      env:
+         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        python ql/misc/scripts/library-coverage/create-pr.py ql "$GITHUB_REPOSITORY"

.github/workflows/csv-coverage.yml

@@ -4,39 +4,46 @@ on:
   workflow_dispatch:
     inputs:
       qlModelShaOverride:
-        description: "github/codeql repo SHA used for looking up the CSV models"
+        description: 'github/codeql repo SHA used for looking up the CSV models'
         required: false
 
 jobs:
   build:
+
     runs-on: ubuntu-latest
 
     steps:
-      - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
-        with:
-          path: script
-      - name: Clone self (github/codeql) for analysis
-        uses: actions/checkout@v3
-        with:
-          path: codeqlModels
-          ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - name: Download CodeQL CLI
-        uses: ./script/.github/actions/fetch-codeql
-      - name: Build modeled package list
-        run: |
-          python script/misc/scripts/library-coverage/generate-report.py ci codeqlModels script
-      - name: Upload CSV package list
-        uses: actions/upload-artifact@v3
-        with:
-          name: framework-coverage-csv
-          path: framework-coverage-*.csv
-      - name: Upload RST package list
-        uses: actions/upload-artifact@v3
-        with:
-          name: framework-coverage-rst
-          path: framework-coverage-*.rst
+    - name: Clone self (github/codeql)
+      uses: actions/checkout@v3
+      with:
+        path: script
+    - name: Clone self (github/codeql) for analysis
+      uses: actions/checkout@v3
+      with:
+        path: codeqlModels
+        ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: 3.8
+    - name: Download CodeQL CLI
+      env:
+         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
+    - name: Unzip CodeQL CLI
+      run: unzip -d codeql-cli codeql-linux64.zip
+    - name: Build modeled package list
+      run: |
+        PATH="$PATH:codeql-cli/codeql" python script/misc/scripts/library-coverage/generate-report.py ci codeqlModels script
+    - name: Upload CSV package list
+      uses: actions/upload-artifact@v3
+      with:
+        name: framework-coverage-csv
+        path: framework-coverage-*.csv
+    - name: Upload RST package list
+      uses: actions/upload-artifact@v3
+      with:
+        name: framework-coverage-rst
+        path: framework-coverage-*.rst
+

.github/workflows/go-tests.yml

@@ -1,114 +0,0 @@
-name: "Go: Run Tests"
-on:
-  pull_request:
-    paths:
-      - "go/**"
-      - .github/workflows/go-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-jobs:
-  test-linux:
-    name: Test Linux (Ubuntu)
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.19
-        id: go
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up CodeQL CLI
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Enable problem matchers in repository
-        shell: bash
-        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-      - name: Build
-        run: |
-          cd go
-          make
-
-      - name: Check that all QL and Go code is autoformatted
-        run: |
-          cd go
-          make check-formatting
-
-      - name: Compile qhelp files to markdown
-        run: |
-          cd go
-          env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown
-
-      - name: Upload qhelp markdown
-        uses: actions/upload-artifact@v2
-        with:
-          name: qhelp-markdown
-          path: go/qhelp-out/**/*.md
-
-      - name: Test
-        run: |
-          cd go
-          make test
-
-  test-mac:
-    name: Test MacOS
-    runs-on: macos-latest
-    steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.19
-        id: go
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up CodeQL CLI
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Enable problem matchers in repository
-        shell: bash
-        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-      - name: Build
-        run: |
-          cd go
-          make
-
-      - name: Test
-        run: |
-          cd go
-          make test
-
-  test-win:
-    name: Test Windows
-    runs-on: windows-2019
-    steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.19
-        id: go
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up CodeQL CLI
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Enable problem matchers in repository
-        shell: bash
-        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-      - name: Build
-        run: |
-          cd go
-          make
-
-      - name: Test
-        run: |
-          cd go
-          make test

.github/workflows/js-ml-tests.yml

@@ -5,8 +5,6 @@ on:
     paths:
       - "javascript/ql/experimental/adaptivethreatmodeling/**"
       - .github/workflows/js-ml-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
@@ -14,9 +12,6 @@ on:
     paths:
       - "javascript/ql/experimental/adaptivethreatmodeling/**"
       - .github/workflows/js-ml-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-  workflow_dispatch:
 
 defaults:
   run:

.github/workflows/labeler.yml

@@ -4,9 +4,6 @@ on:
 
 jobs:
   triage:
-    permissions:
-      contents: read
-      pull-requests: write
     runs-on: ubuntu-latest
     steps:
     - uses: actions/labeler@v4

.github/workflows/mad_modelDiff.yml

@@ -61,7 +61,7 @@ jobs:
             DATABASE=$2
             cd codeql-$QL_VARIANT
             SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/model-generator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $MODELS/${SHORTNAME}.qll
+            python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
             mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
             cd ..
           }

.github/workflows/mad_regenerate-models.yml

@@ -9,7 +9,6 @@ on:
       - main
     paths:
       - ".github/workflows/mad_regenerate-models.yml"
-      - ".github/actions/fetch-codeql/action.yml"
 
 jobs:
   regenerate-models:
@@ -21,7 +20,7 @@ jobs:
         ref: ["placeholder"]
         include:
           - slug: "apache/commons-io"
-            ref: "13258ce2d07aa0e764bbaa8020af4dcd3a02a620"
+            ref: "8985de8fe74f6622a419b37a6eed0dbc484dc128"
         exclude:
           - slug: "placeholder"
             ref: "placeholder"

.github/workflows/ql-for-ql-build.yml

@@ -5,28 +5,21 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
-    paths:
-      - "ql/**"
-      - "**.qll"
-      - "**.ql"
-      - "**.dbscheme"
-      - "**/qlpack.yml"
-      - ".github/workflows/ql-for-ql-build.yml"
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  analyze:
-    runs-on: ubuntu-latest-xl
+  queries:
+    runs-on: ubuntu-latest
     steps:
-      ### Build the queries ###
       - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with:
           languages: javascript # does not matter
+          tools: latest
       - name: Get CodeQL version
         id: get-codeql-version
         run: |
@@ -34,37 +27,37 @@ jobs:
         shell: bash
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Cache entire pack
-        id: cache-pack
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/pack
-          key: ${{ runner.os }}-pack-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
       - name: Cache queries
-        if: steps.cache-pack.outputs.cache-hit != 'true'
         id: cache-queries
         uses: actions/cache@v3
         with:
-          path: ${{ runner.temp }}/queries
-          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
+          path: ${{ runner.temp }}/query-pack.zip
+          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
       - name: Build query pack
-        if: steps.cache-queries.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        if: steps.cache-queries.outputs.cache-hit != 'true'
         run: |
           cd ql/ql/src
-          "${CODEQL}" pack create -j 16
-          mv .codeql/pack/codeql/ql/0.0.0 ${{ runner.temp }}/queries
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Move cache queries to pack
-        if: steps.cache-pack.outputs.cache-hit != 'true'
-        run: |
-          cp -r ${{ runner.temp }}/queries ${{ runner.temp }}/pack
+          "${CODEQL}" pack create
+          cd .codeql/pack/codeql/ql/0.0.0
+          zip "${PACKZIP}" -r .
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+          PACKZIP: ${{ runner.temp }}/query-pack.zip
+      - name: Upload query pack
+        uses: actions/upload-artifact@v3
+        with:
+          name: query-pack-zip
+          path: ${{ runner.temp }}/query-pack.zip
 
-      ### Build the extractor ###
+  extractors:
+    strategy:
+      fail-fast: false
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
       - name: Cache entire extractor
-        if: steps.cache-pack.outputs.cache-hit != 'true'
         id: cache-extractor
         uses: actions/cache@v3
         with:
@@ -75,7 +68,7 @@ jobs:
             ql/target/release/ql-extractor.exe
           key: ${{ runner.os }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
       - name: Cache cargo
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         uses: actions/cache@v3
         with:
           path: |
@@ -84,87 +77,127 @@ jobs:
             ql/target
           key: ${{ runner.os }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Check formatting
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd ql; cargo fmt --all -- --check
       - name: Build
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd ql; cargo build --verbose
       - name: Run tests
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd ql; cargo test --verbose
       - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd ql; cargo build --release
       - name: Generate dbscheme
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
+      - uses: actions/upload-artifact@v3
+        with:
+          name: extractor-ubuntu-latest
+          path: |
+            ql/target/release/ql-autobuilder
+            ql/target/release/ql-autobuilder.exe
+            ql/target/release/ql-extractor
+            ql/target/release/ql-extractor.exe
+          retention-days: 1
+  package:
+    runs-on: ubuntu-latest
 
-      ### Package the queries and extractor ###
-      - name: Package pack
-        if: steps.cache-pack.outputs.cache-hit != 'true'
+    needs:
+      - extractors
+      - queries
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: query-pack-zip
+          path: query-pack-zip
+      - uses: actions/download-artifact@v3
+        with:
+          name: extractor-ubuntu-latest
+          path: linux64
+      - run: |
+          unzip query-pack-zip/*.zip -d pack
+          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats pack/
+          mkdir -p pack/tools/linux64
+          if [[ -f linux64/ql-autobuilder ]]; then
+            cp linux64/ql-autobuilder pack/tools/linux64/autobuilder
+            chmod +x pack/tools/linux64/autobuilder
+          fi
+          if [[ -f linux64/ql-extractor ]]; then
+            cp linux64/ql-extractor pack/tools/linux64/extractor
+            chmod +x pack/tools/linux64/extractor
+          fi
+          cd pack
+          zip -rq ../codeql-ql.zip .
+      - uses: actions/upload-artifact@v3
+        with:
+          name: codeql-ql-pack
+          path: codeql-ql.zip
+          retention-days: 1
+  analyze:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        folder: [cpp, csharp, java, javascript, python, ql, ruby, swift]
+
+    needs:
+      - package
+
+    steps:
+      - name: Download pack
+        uses: actions/download-artifact@v3
+        with:
+          name: codeql-ql-pack
+          path: ${{ runner.temp }}/codeql-ql-pack-artifact
+
+      - name: Prepare pack
         run: |
-          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats ${PACK}/
-          mkdir -p ${PACK}/tools/linux64
-          cp ql/target/release/ql-autobuilder  ${PACK}/tools/linux64/autobuilder
-          cp ql/target/release/ql-extractor ${PACK}/tools/linux64/extractor
-          chmod +x ${PACK}/tools/linux64/autobuilder
-          chmod +x ${PACK}/tools/linux64/extractor
+          unzip "${PACK_ARTIFACT}/*.zip" -d "${PACK}"
         env:
+          PACK_ARTIFACT: ${{ runner.temp }}/codeql-ql-pack-artifact
           PACK: ${{ runner.temp }}/pack
-
-      ### Run the analysis ###
       - name: Hack codeql-action options
         run: |
-          JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .resolve.languages=["--search-path", $pack] | .database.init=["--search-path", $pack]')
+          JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
           echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
         env:
           PACK: ${{ runner.temp }}/pack
 
+      - name: Checkout repository
+        uses: actions/checkout@v3
       - name: Create CodeQL config file
         run: |
+          echo "paths:" > ${CONF}
+          echo "  - ${FOLDER}" >> ${CONF}
           echo "paths-ignore:" >> ${CONF}
-          echo "  - ql/ql/test" >> ${CONF}
-          echo "  - \"*/ql/lib/upgrades/\"" >> ${CONF}
+          echo "  - ql/ql/test" >> ${CONF} 
           echo "disable-default-queries: true" >> ${CONF}
-          echo "queries:" >> ${CONF}
-          echo "  - uses: ./ql/ql/src/codeql-suites/ql-code-scanning.qls" >> ${CONF}
+          echo "packs:" >> ${CONF}
+          echo "  - codeql/ql" >> ${CONF}
           echo "Config file: "
           cat ${CONF}
-        env:
+        env: 
           CONF: ./ql-for-ql-config.yml
+          FOLDER: ${{ matrix.folder }}
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with:
           languages: ql
           db-location: ${{ runner.temp }}/db
           config-file: ./ql-for-ql-config.yml
-      - name: Move pack cache
-        run: |
-          cp -r ${PACK}/.cache ql/ql/src/.cache
-        env:
-          PACK: ${{ runner.temp }}/pack
+          tools: latest
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
-        with:
-          category: "ql-for-ql"
+        uses: github/codeql-action/analyze@aa93aea877e5fb8841bcb1193f672abf6e9f2980
+        with: 
+          category: "ql-for-ql-${{ matrix.folder }}"
       - name: Copy sarif file to CWD
-        run: cp ../results/ql.sarif ./ql-for-ql.sarif
-      - name: Fixup the $scema in sarif  # Until https://github.com/microsoft/sarif-vscode-extension/pull/436/ is part in a stable release
-        run: |
-          sed -i 's/\$schema.*/\$schema": "https:\/\/raw.githubusercontent.com\/oasis-tcs\/sarif-spec\/master\/Schemata\/sarif-schema-2.1.0",/' ql-for-ql.sarif
+        run: cp ../results/ql.sarif ./${{ matrix.folder }}.sarif
       - name: Sarif as artifact
         uses: actions/upload-artifact@v3
         with:
-          name: ql-for-ql.sarif
-          path: ql-for-ql.sarif
-      - name: Split out the sarif file into langs
-        run: |
-          mkdir split-sarif
-          node ./ql/scripts/split-sarif.js ql-for-ql.sarif split-sarif
-      - name: Upload langs as artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: ql-for-ql-langs
-          path: split-sarif
-          retention-days: 1
+          name: ${{ matrix.folder }}.sarif
+          path: ${{ matrix.folder }}.sarif
+

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -19,13 +19,14 @@ jobs:
       matrix:
         repo:
           - github/codeql
+          - github/codeql-go
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v3
@@ -36,7 +37,7 @@ jobs:
             ql/target
           key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Build Extractor
-        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./scripts/create-extractor-pack.sh
+        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Checkout ${{ matrix.repo }}
@@ -76,7 +77,7 @@ jobs:
           path: stats
       - run: |
           python -m pip install --user lxml
-          find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ruby/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
+          find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ql/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
       - uses: actions/upload-artifact@v3
         with:
           name: ql.dbscheme.stats

.github/workflows/ql-for-ql-tests.yml

@@ -5,12 +5,10 @@ on:
     branches: [main]
     paths:
       - "ql/**"
-      - codeql-workspace.yml
   pull_request:
     branches: [main]
     paths:
       - "ql/**"
-      - codeql-workspace.yml
 
 env:
   CARGO_TERM_COLOR: always
@@ -22,7 +20,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v3
@@ -36,7 +34,7 @@ jobs:
         run: |
           cd ql;
           codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
-          env "PATH=$PATH:$codeqlpath" ./scripts/create-extractor-pack.sh
+          env "PATH=$PATH:$codeqlpath" ./create-extractor-pack.sh
       - name: Run QL tests
         run: |
           "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test
@@ -44,7 +42,7 @@ jobs:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Check QL formatting
         run: |
-          find ql/ql/src "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
+          find ql/ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Check QL compilation

.github/workflows/query-list.yml

@@ -5,12 +5,9 @@ on:
     branches:
      - main
      - 'rc/**'
-    tags:
-     - 'codeql-cli/*'
   pull_request:
     paths:
       - '.github/workflows/query-list.yml'
-      - '.github/actions/fetch-codeql/action.yml'
       - 'misc/scripts/generate-code-scanning-query-list.py'
 
 jobs:
@@ -23,18 +20,30 @@ jobs:
       uses: actions/checkout@v3
       with:
         path: codeql 
+    - name: Clone github/codeql-go
+      uses: actions/checkout@v3
+      with:
+        repository: 'github/codeql-go'
+        path: codeql-go
     - name: Set up Python 3.8
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v3
       with:
         python-version: 3.8
     - name: Download CodeQL CLI
-      # Look under the `codeql` directory, as this is where we checked out the `github/codeql` repo
-      uses: ./codeql/.github/actions/fetch-codeql
+      uses: dsaltares/fetch-gh-release-asset@aa37ae5c44d3c9820bc12fe675e8670ecd93bd1c
+      with:
+        repo: "github/codeql-cli-binaries"
+        version: "latest"
+        file: "codeql-linux64.zip"
+        token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Unzip CodeQL CLI
+      run: unzip -d codeql-cli codeql-linux64.zip
     - name: Build code scanning query list
       run: |
-        python codeql/misc/scripts/generate-code-scanning-query-list.py > code-scanning-query-list.csv
+        PATH="$PATH:codeql-cli/codeql" python codeql/misc/scripts/generate-code-scanning-query-list.py > code-scanning-query-list.csv
     - name: Upload code scanning query list
       uses: actions/upload-artifact@v3
       with:
         name: code-scanning-query-list
         path: code-scanning-query-list.csv
+    

.github/workflows/ruby-build.yml

@@ -5,8 +5,6 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-build.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
@@ -14,8 +12,6 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-build.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
@@ -92,14 +88,19 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Fetch CodeQL
-        uses: ./.github/actions/fetch-codeql
+        run: |
+          LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
+          gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
+          unzip -q codeql-linux64.zip
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
       - name: Build Query Pack
         run: |
-          codeql pack create ql/lib --output target/packs
-          codeql pack install ql/src
-          codeql pack create ql/src --output target/packs
+          codeql/codeql pack create ql/lib --output target/packs
+          codeql/codeql pack install ql/src
+          codeql/codeql pack create ql/src --output target/packs
           PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
-          codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
+          codeql/codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
           (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
       - uses: actions/upload-artifact@v3
         with:
@@ -176,15 +177,19 @@ jobs:
     runs-on: ${{ matrix.os }}
     needs: [package]
     steps:
-      - uses: actions/checkout@v3
-      - name: Fetch CodeQL
-        uses: ./.github/actions/fetch-codeql
-
       - uses: actions/checkout@v3
         with:
           repository: Shopify/example-ruby-app
           ref: 67a0decc5eb550f3a9228eda53925c3afd40dfe9
-
+      - name: Fetch CodeQL
+        shell: bash
+        run: |
+          LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
+          gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql.zip "$LATEST"
+          unzip -q codeql.zip
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        working-directory: ${{ runner.temp }}
       - name: Download Ruby bundle
         uses: actions/download-artifact@v3
         with:
@@ -208,12 +213,12 @@ jobs:
       - name: Run QL test
         shell: bash
         run: |
-          codeql test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
+          "${{ runner.temp }}/codeql/codeql" test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
       - name: Create database
         shell: bash
         run: |
-          codeql database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
+          "${{ runner.temp }}/codeql/codeql" database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
       - name: Analyze database
         shell: bash
         run: |
-          codeql database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls
+          "${{ runner.temp }}/codeql/codeql" database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/ruby-qltest.yml

@@ -5,8 +5,6 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-qltest.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
@@ -14,8 +12,6 @@ on:
     paths:
       - "ruby/**"
       - .github/workflows/ruby-qltest.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"

.github/workflows/swift-codegen.yml

@@ -5,7 +5,6 @@ on:
     paths:
       - "swift/**"
       - .github/workflows/swift-codegen.yml
-      - .github/actions/fetch-codeql/action.yml
     branches:
       - main
 
@@ -17,21 +16,17 @@ jobs:
       - uses: ./.github/actions/fetch-codeql
       - uses: bazelbuild/setup-bazelisk@v2
       - uses: actions/setup-python@v3
-      - uses: pre-commit/action@v3.0.0
-        name: Check that python code is properly formatted
         with:
-          extra_args: autopep8 --all-files
+          python-version: '~3.8'
+          cache: 'pip'
+      - name: Install python dependencies
+        run: |
+          pip install -r swift/codegen/requirements.txt
       - name: Run unit tests
         run: |
           bazel test //swift/codegen/test --test_output=errors
-      - uses: pre-commit/action@v3.0.0
-        name: Check that QL generated code was checked in
-        with:
-          extra_args: swift-codegen --all-files
-      - name: Generate C++ files
+      - name: Check that code was generated
         run: |
-          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-cpp-files
-      - uses: actions/upload-artifact@v3
-        with:
-          name: swift-generated-cpp-files
-          path: swift-generated-cpp-files/**
+          bazel run //swift/codegen
+          git add swift
+          git diff --exit-code --stat HEAD

.github/workflows/swift-integration-tests.yml

@@ -1,35 +0,0 @@
-name: "Swift: Run Integration Tests"
-
-on:
-  pull_request:
-    paths:
-      - "swift/**"
-      - .github/workflows/swift-integration-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-defaults:
-  run:
-    working-directory: swift
-
-jobs:
-  integration-tests:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-          - ubuntu-20.04
-#          - macos-latest  TODO
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - uses: bazelbuild/setup-bazelisk@v2
-      - uses: actions/setup-python@v3
-      - name: Build Swift extractor
-        run: |
-          bazel run //swift:create-extractor-pack
-      - name: Run integration tests
-        run: |
-          python integration-tests/runner.py

.github/workflows/swift-qltest.yml

@@ -5,8 +5,6 @@ on:
     paths:
       - "swift/**"
       - .github/workflows/swift-qltest.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
     branches:
       - main
 defaults:
@@ -31,6 +29,13 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '~3.8'
+          cache: 'pip'
+      - name: Install python dependencies
+        run: |
+          pip install -r codegen/requirements.txt
       - name: Build Swift extractor
         run: |
           bazel run //swift:create-extractor-pack

.github/workflows/validate-change-notes.yml

@@ -5,7 +5,6 @@ on:
     paths:
       - "*/ql/*/change-notes/**/*"
       - ".github/workflows/validate-change-notes.yml"
-      - ".github/actions/fetch-codeql/action.yml"
     branches:
       - main
       - "rc/*"
@@ -13,7 +12,6 @@ on:
     paths:
       - "*/ql/*/change-notes/**/*"
       - ".github/workflows/validate-change-notes.yml"
-      - ".github/actions/fetch-codeql/action.yml"
 
 jobs:
   check-change-note:

.gitignore

@@ -9,7 +9,6 @@
 # qltest projects and artifacts
 */ql/test/**/*.testproj
 */ql/test/**/*.actual
-*/ql/test/**/go.sum
 
 # Visual studio temporaries, except a file used by QL4VS
 .vs/*
@@ -38,26 +37,5 @@ csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
 # links created by bazel
 /bazel-*
 
-# local bazel options
-/local.bazelrc
-
 # CLion project files
 /.clwb
-
-# Go build artifacts
-go/build/*
-
-# Go binaries
-go/tools/bin
-go/tools/linux64
-go/tools/osx64
-go/tools/win64
-go/tools/tokenizer.jar
-go/main
-
-# node_modules folders except in the JS test suite
-node_modules/
-!/javascript/ql/test/**/node_modules/
-
-# Temporary folders for working with generated models
-.model-temp

.lgtm.yml

@@ -6,7 +6,6 @@ path_classifiers:
   test:
   - csharp/ql/src
   - csharp/ql/test
-  - go/ql/test
   - javascript/extractor/parser-tests
   - javascript/extractor/tests
   - javascript/ql/src
@@ -14,9 +13,6 @@ path_classifiers:
   - python/ql/src
   - python/ql/test
 
-  example:
-  - go/ql/src
-
 queries:
   - include: "*"
 

.pre-commit-config.yaml

@@ -15,12 +15,6 @@ repos:
       - id: clang-format
         files: ^swift/.*\.(h|c|cpp)$
 
-  - repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v1.6.0
-    hooks:
-      - id: autopep8
-        files: ^swift/codegen/.*\.py
-
   - repo: local
     hooks:
       - id: codeql-format
@@ -31,7 +25,7 @@ repos:
 
       - id: sync-files
         name: Fix files required to be identical
-        files: \.(qll?|qhelp|swift)$
+        files: \.(qll?|qhelp)$
         language: system
         entry: python3 config/sync-files.py --latest
         pass_filenames: false
@@ -46,7 +40,7 @@ repos:
         name: Run Swift checked in code generation
         files: ^swift/(codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements))
         language: system
-        entry: bazel run //swift/codegen -- --quiet
+        entry: bazel run //swift/codegen
         pass_filenames: false
 
       - id: swift-codegen-unit-tests

CODEOWNERS

@@ -1,13 +1,9 @@
 /cpp/ @github/codeql-c-analysis
 /csharp/ @github/codeql-csharp
-/go/ @github/codeql-go
 /java/ @github/codeql-java
 /javascript/ @github/codeql-javascript
 /python/ @github/codeql-python
 /ruby/ @github/codeql-ruby
-/swift/ @github/codeql-c
-/java/kotlin-extractor/ @github/codeql-kotlin
-/java/kotlin-explorer/ @github/codeql-kotlin
 
 # ML-powered queries
 /javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
@@ -28,8 +24,8 @@
 # QL for QL reviewers
 /ql/ @github/codeql-ql-for-ql-reviewers
 
-# Bazel (excluding BUILD.bazel files)
-WORKSPACE.bazel @github/codeql-ci-reviewers
+# Bazel
+**/*.bazel @github/codeql-ci-reviewers
 **/*.bzl @github/codeql-ci-reviewers
 
 # Documentation etc
@@ -38,8 +34,6 @@ WORKSPACE.bazel @github/codeql-ci-reviewers
 
 # Workflows
 /.github/workflows/ @github/codeql-ci-reviewers
-/.github/workflows/go-* @github/codeql-go
 /.github/workflows/js-ml-tests.yml @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
 /.github/workflows/ruby-* @github/codeql-ruby
-/.github/workflows/swift-* @github/codeql-c

README.md

@@ -1,6 +1,6 @@
 # CodeQL
 
-This open source repository contains the standard CodeQL libraries and queries that power [GitHub Advanced Security](https://github.com/features/security/code) and the other application security products that [GitHub](https://github.com/features/security/) makes available to its customers worldwide.
+This open source repository contains the standard CodeQL libraries and queries that power [GitHub Advanced Security](https://github.com/features/security/code) and the other application security products that [GitHub](https://github.com/features/security/) makes available to its customers worldwide. For the queries, libraries, and extractor that power Go analysis, visit the [CodeQL for Go repository](https://github.com/github/codeql-go).
 
 ## How do I learn CodeQL and run queries?
 

codeql-workspace.yml

@@ -1,32 +0,0 @@
-provide:
-  - "*/ql/src/qlpack.yml"
-  - "*/ql/lib/qlpack.yml"
-  - "*/ql/test/qlpack.yml"
-  - "*/ql/examples/qlpack.yml"
-  - "*/ql/consistency-queries/qlpack.yml"
-  - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
-  - "go/ql/config/legacy-support/qlpack.yml"
-  - "go/build/codeql-extractor-go/codeql-extractor.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml"
-  # This pack is explicitly excluded from the workspace since most users
-  # will want to use a version of this pack from the package cache. Internal
-  # users can uncomment the following line and place a custom ML model
-  # in the corresponding pack to test a custom ML model within their local
-  # checkout.
-  # - "javascript/ql/experimental/adaptivethreatmodeling/model/qlpack.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml"
-  - "csharp/ql/campaigns/Solorigate/lib/qlpack.yml"
-  - "csharp/ql/campaigns/Solorigate/src/qlpack.yml"
-  - "csharp/ql/campaigns/Solorigate/test/qlpack.yml"
-  - "misc/legacy-support/*/qlpack.yml"
-  - "misc/suite-helpers/qlpack.yml"
-  - "ruby/extractor-pack/codeql-extractor.yml"
-  - "swift/extractor-pack/codeql-extractor.yml"
-  - "ql/extractor-pack/codeql-extractor.ym"
-
-versionPolicies:
-  default:
-    requireChangeNotes: true
-    committedPrereleaseSuffix: dev
-    committedVersion: nextPatchRelease

config/identical-files.json

@@ -22,15 +22,13 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplForContentDataFlow.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForLibraries.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForLibraries.qll"
   ],
   "DataFlow Java/C++/C#/Python Common": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll",
@@ -38,8 +36,7 @@
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll"
   ],
   "TaintTracking::Configuration Java/C++/C#/Python": [
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
@@ -60,8 +57,7 @@
     "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking3/TaintTrackingImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking4/TaintTrackingImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttrackingforlibraries/TaintTrackingImpl.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttrackingforlibraries/TaintTrackingImpl.qll"
   ],
   "DataFlow Java/C++/C#/Python Consistency checks": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll",
@@ -69,14 +65,12 @@
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplConsistency.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplConsistency.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplConsistency.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplConsistency.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplConsistency.qll"
   ],
   "DataFlow Java/C# Flow Summaries": [
     "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/FlowSummaryImpl.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/FlowSummaryImpl.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/FlowSummaryImpl.qll"
   ],
   "SsaReadPosition Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll",
@@ -391,9 +385,7 @@
     "java/ql/test/TestUtilities/InlineExpectationsTest.qll",
     "python/ql/test/TestUtilities/InlineExpectationsTest.qll",
     "ruby/ql/test/TestUtilities/InlineExpectationsTest.qll",
-    "ql/ql/test/TestUtilities/InlineExpectationsTest.qll",
-    "go/ql/test/TestUtilities/InlineExpectationsTest.qll",
-    "swift/ql/test/TestUtilities/InlineExpectationsTest.qll"
+    "ql/ql/test/TestUtilities/InlineExpectationsTest.qll"
   ],
   "C++ ExternalAPIs": [
     "cpp/ql/src/Security/CWE/CWE-020/ExternalAPIs.qll",
@@ -454,17 +446,19 @@
     "python/ql/src/Lexical/CommentedOutCodeReferences.inc.qhelp"
   ],
   "IDE Contextual Queries": [
-    "cpp/ql/lib/IDEContextual.qll",
-    "csharp/ql/lib/IDEContextual.qll",
-    "java/ql/lib/IDEContextual.qll",
-    "javascript/ql/lib/IDEContextual.qll",
-    "python/ql/lib/analysis/IDEContextual.qll"
+    "cpp/ql/src/IDEContextual.qll",
+    "csharp/ql/src/IDEContextual.qll",
+    "java/ql/src/IDEContextual.qll",
+    "javascript/ql/src/IDEContextual.qll",
+    "python/ql/src/analysis/IDEContextual.qll"
   ],
   "SSA C#": [
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/SsaImplCommon.qll",
+    "csharp/ql/lib/semmle/code/csharp/controlflow/internal/pressa/SsaImplCommon.qll",
+    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/basessa/SsaImplCommon.qll",
+    "csharp/ql/lib/semmle/code/cil/internal/SsaImplCommon.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/SsaImplCommon.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaImplCommon.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/SsaImplCommon.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaImplCommon.qll"
   ],
   "CryptoAlgorithms Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/CryptoAlgorithms.qll",
@@ -481,44 +475,29 @@
     "python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll",
     "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll"
   ],
-  "ReDoS Util Python/JS/Ruby/Java": [
-    "javascript/ql/lib/semmle/javascript/security/regexp/NfaUtils.qll",
-    "python/ql/lib/semmle/python/security/regexp/NfaUtils.qll",
-    "ruby/ql/lib/codeql/ruby/security/regexp/NfaUtils.qll",
-    "java/ql/lib/semmle/code/java/security/regexp/NfaUtils.qll"
+  "ReDoS Util Python/JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll",
+    "python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll",
+    "ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll"
   ],
-  "ReDoS Exponential Python/JS/Ruby/Java": [
-    "javascript/ql/lib/semmle/javascript/security/regexp/ExponentialBackTracking.qll",
-    "python/ql/lib/semmle/python/security/regexp/ExponentialBackTracking.qll",
-    "ruby/ql/lib/codeql/ruby/security/regexp/ExponentialBackTracking.qll",
-    "java/ql/lib/semmle/code/java/security/regexp/ExponentialBackTracking.qll"
+  "ReDoS Exponential Python/JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/performance/ExponentialBackTracking.qll",
+    "python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll",
+    "ruby/ql/lib/codeql/ruby/security/performance/ExponentialBackTracking.qll"
   ],
-  "ReDoS Polynomial Python/JS/Ruby/Java": [
-    "javascript/ql/lib/semmle/javascript/security/regexp/SuperlinearBackTracking.qll",
-    "python/ql/lib/semmle/python/security/regexp/SuperlinearBackTracking.qll",
-    "ruby/ql/lib/codeql/ruby/security/regexp/SuperlinearBackTracking.qll",
-    "java/ql/lib/semmle/code/java/security/regexp/SuperlinearBackTracking.qll"
-  ],
-  "RegexpMatching Python/JS/Ruby": [
-    "javascript/ql/lib/semmle/javascript/security/regexp/RegexpMatching.qll",
-    "python/ql/lib/semmle/python/security/regexp/RegexpMatching.qll",
-    "ruby/ql/lib/codeql/ruby/security/regexp/RegexpMatching.qll"
+  "ReDoS Polynomial Python/JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll",
+    "python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll",
+    "ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll"
   ],
   "BadTagFilterQuery Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll",
     "python/ql/lib/semmle/python/security/BadTagFilterQuery.qll",
     "ruby/ql/lib/codeql/ruby/security/BadTagFilterQuery.qll"
   ],
-  "OverlyLargeRange Python/JS/Ruby/Java": [
-    "javascript/ql/lib/semmle/javascript/security/OverlyLargeRangeQuery.qll",
-    "python/ql/lib/semmle/python/security/OverlyLargeRangeQuery.qll",
-    "ruby/ql/lib/codeql/ruby/security/OverlyLargeRangeQuery.qll",
-    "java/ql/lib/semmle/code/java/security/OverlyLargeRangeQuery.qll"
-  ],
   "CFG": [
     "csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll",
-    "ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll",
-    "swift/ql/lib/codeql/swift/controlflow/internal/ControlFlowGraphImplShared.qll"
+    "ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll"
   ],
   "TypeTracker": [
     "python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll",
@@ -536,9 +515,7 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/AccessPathSyntax.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/AccessPathSyntax.qll",
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/AccessPathSyntax.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll",
-    "python/ql/lib/semmle/python/frameworks/data/internal/AccessPathSyntax.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/AccessPathSyntax.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll"
   ],
   "IncompleteUrlSubstringSanitization": [
     "javascript/ql/src/Security/CWE-020/IncompleteUrlSubstringSanitization.qll",
@@ -556,8 +533,7 @@
   ],
   "ApiGraphModels": [
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll",
-    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll",
-    "python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll"
+    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll"
   ],
   "TaintedFormatStringQuery Ruby/JS": [
     "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringQuery.qll",
@@ -574,29 +550,5 @@
   "HttpToFileAccessCustomizations JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/dataflow/HttpToFileAccessCustomizations.qll",
     "ruby/ql/lib/codeql/ruby/security/HttpToFileAccessCustomizations.qll"
-  ],
-  "Typo database": [
-    "javascript/ql/src/Expressions/TypoDatabase.qll",
-    "ql/ql/src/codeql_ql/style/TypoDatabase.qll"
-  ],
-  "Swift declarations test file": [
-    "swift/ql/test/extractor-tests/declarations/declarations.swift",
-    "swift/ql/test/library-tests/ast/declarations.swift"
-  ],
-  "Swift statements test file": [
-    "swift/ql/test/extractor-tests/statements/statements.swift",
-    "swift/ql/test/library-tests/ast/statements.swift"
-  ],
-  "Swift expressions test file": [
-    "swift/ql/test/extractor-tests/expressions/expressions.swift",
-    "swift/ql/test/library-tests/ast/expressions.swift"
-  ],
-  "Swift patterns test file": [
-    "swift/ql/test/extractor-tests/patterns/patterns.swift",
-    "swift/ql/test/library-tests/ast/patterns.swift"
-  ],
-  "IncompleteMultiCharacterSanitization JS/Ruby": [
-    "javascript/ql/lib/semmle/javascript/security/IncompleteMultiCharacterSanitizationQuery.qll",
-    "ruby/ql/lib/codeql/ruby/security/IncompleteMultiCharacterSanitizationQuery.qll"
   ]
-}
+}

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -299,7 +299,7 @@ namespace Semmle.Autobuild.Cpp.Tests
         {
             Actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test.sln -DisableParallelProcessing"] = 1;
             Actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test.sln -DisableParallelProcessing"] = 0;
-            Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0;
+            Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /p:UseSharedCompilation=false /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0;
             Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0;

cpp/downgrades/19e31bf071f588bb7efd1e4d5a185ce4f6fbbd84/upgrade.properties

@@ -1,3 +0,0 @@
-description: Add relation for tracking C++ braced initializers
-compatibility: full
-braced_initialisers.rel: delete

cpp/downgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/exprs.ql

@@ -1,17 +0,0 @@
-class Expr extends @expr {
-  string toString() { none() }
-}
-
-class Location extends @location_expr {
-  string toString() { none() }
-}
-
-predicate isExprWithNewBuiltin(Expr expr) {
-  exists(int kind | exprs(expr, kind, _) | 330 <= kind and kind <= 334)
-}
-
-from Expr expr, int kind, int kind_new, Location location
-where
-  exprs(expr, kind, location) and
-  if isExprWithNewBuiltin(expr) then kind_new = 0 else kind_new = kind
-select expr, kind_new, location

cpp/downgrades/23f7cbb88a4eb29f30c3490363dc201bc054c5ff/upgrade.properties

@@ -1,3 +0,0 @@
-description: Add new builtin operations
-compatibility: partial
-exprs.rel: run exprs.qlo

cpp/downgrades/34549c3b0937002f11037d01822ebe99442c1402/attribute_args.ql

@@ -1,17 +0,0 @@
-class AttributeArgument extends @attribute_arg {
-  string toString() { none() }
-}
-
-class Attribute extends @attribute {
-  string toString() { none() }
-}
-
-class LocationDefault extends @location_default {
-  string toString() { none() }
-}
-
-from AttributeArgument arg, int kind, Attribute attr, int index, LocationDefault location
-where
-  attribute_args(arg, kind, attr, index, location) and
-  not arg instanceof @attribute_arg_constant_expr
-select arg, kind, attr, index, location

cpp/downgrades/34549c3b0937002f11037d01822ebe99442c1402/upgrade.properties

@@ -1,4 +0,0 @@
-description: Support all constant attribute arguments
-compatibility: backwards
-attribute_arg_constant.rel: delete
-attribute_args.rel: run attribute_args.qlo

cpp/downgrades/73af5058c6899dcdb05754c27ca966aeb3a68c94/exprs.ql

@@ -1,13 +0,0 @@
-class Expr extends @expr {
-  string toString() { none() }
-}
-
-class Location extends @location_expr {
-  string toString() { none() }
-}
-
-from Expr expr, int kind, int kind_new, Location location
-where
-  exprs(expr, kind, location) and
-  if expr instanceof @blockassignexpr then kind_new = 0 else kind_new = kind
-select expr, kind_new, location

cpp/downgrades/73af5058c6899dcdb05754c27ca966aeb3a68c94/upgrade.properties

@@ -1,3 +0,0 @@
-description: Support block assignment
-compatibility: partial
-exprs.rel: run exprs.qlo

cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/exprparents.ql

@@ -1,21 +0,0 @@
-class Element extends @element {
-  string toString() { none() }
-}
-
-class Expr extends @expr {
-  string toString() { none() }
-}
-
-class Stmt extends @stmt {
-  string toString() { none() }
-}
-
-predicate isStmtWithInitializer(Stmt stmt) {
-  exists(int kind | stmts(stmt, kind, _) | kind = 2 or kind = 11 or kind = 35)
-}
-
-from Expr child, int index, int index_new, Element parent
-where
-  exprparents(child, index, parent) and
-  if isStmtWithInitializer(parent) then index_new = index - 1 else index_new = index
-select child, index_new, parent

cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/stmtparents.ql

@@ -1,22 +0,0 @@
-class Element extends @element {
-  string toString() { none() }
-}
-
-class Stmt extends @stmt {
-  string toString() { none() }
-}
-
-predicate isStmtWithInitializer(Stmt stmt) {
-  exists(int kind | stmts(stmt, kind, _) | kind = 2 or kind = 11 or kind = 35)
-}
-
-from Stmt child, int index, int index_new, Element parent
-where
-  stmtparents(child, index, parent) and
-  (
-    not isStmtWithInitializer(parent)
-    or
-    index > 0
-  ) and
-  if isStmtWithInitializer(parent) then index_new = index - 1 else index_new = index
-select child, index_new, parent

cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/upgrade.properties

@@ -1,6 +0,0 @@
-description: Support C++17 if and switch initializers
-compatibility: partial
-if_initialization.rel: delete
-switch_initialization.rel: delete
-exprparents.rel: run exprparents.qlo
-stmtparents.rel: run stmtparents.qlo

cpp/downgrades/f96ad9b2da43bbc9e55a72a165febd270ae07981/upgrade.properties

@@ -1,3 +0,0 @@
-description: Add relation for orphaned local variables
-compatibility: full
-orphaned_variables.rel: delete

cpp/ql/lib/CHANGELOG.md

@@ -1,82 +1,3 @@
-## 0.3.4
-
-### Deprecated APIs
-
-* Many classes/predicates/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
-  The old name still exists as a deprecated alias.
-
-### New Features
-
-* Added support for getting the link targets of global and namespace variables.
-* Added a `BlockAssignExpr` class, which models a `memcpy`-like operation used in compiler generated copy/move constructors and assignment operations.
-
-### Minor Analysis Improvements
-
-* All deprecated predicates/classes/modules that have been deprecated for over a year have been deleted.
-
-## 0.3.3
-
-### New Features
-
-* Added a predicate `getValueConstant` to `AttributeArgument` that yields the argument value as an `Expr` when the value is a constant expression.
-* A new class predicate `MustFlowConfiguration::allowInterproceduralFlow` has been added to the `semmle.code.cpp.ir.dataflow.MustFlow` library. The new predicate can be overridden to disable interprocedural flow.
-* Added subclasses of `BuiltInOperations` for `__builtin_bit_cast`, `__builtin_shuffle`, `__has_unique_object_representations`, `__is_aggregate`, and `__is_assignable`.
-
-### Major Analysis Improvements
-
-* The IR dataflow library now includes flow through global variables. This enables new findings in many scenarios.
-
-## 0.3.2
-
-### Bug Fixes
-
-* Under certain circumstances a variable declaration that is not also a definition could be associated with a `Variable` that did not have the definition as a `VariableDeclarationEntry`. This is now fixed, and a unique `Variable` will exist that has both the declaration and the definition as a `VariableDeclarationEntry`.
-
-## 0.3.1
-
-### Minor Analysis Improvements
-
-* `AnalysedExpr::isNullCheck` and `AnalysedExpr::isValidCheck` have been updated to handle variable accesses on the left-hand side of the C++ logical "and", and variable declarations in conditions.
-
-## 0.3.0
-
-### Deprecated APIs
-
-* The `BarrierGuard` class has been deprecated. Such barriers and sanitizers can now instead be created using the new `BarrierGuard` parameterized module.
-
-### Bug Fixes
-
-* `UserType.getADeclarationEntry()` now yields all forward declarations when the user type is a `class`, `struct`, or `union`.
-
-## 0.2.3
-
-### New Features
-
-* An `isBraced` predicate was added to the `Initializer` class which holds when a C++ braced initializer was used in the initialization.
-
-## 0.2.2
-
-### Deprecated APIs
-
- * The `AnalysedString` class in the `StringAnalysis` module has been replaced with `AnalyzedString`, to follow our style guide. The old name still exists as a deprecated alias.
-
-### New Features
-
-* A `getInitialization` predicate was added to the `ConstexprIfStmt`, `IfStmt`, and `SwitchStmt` classes that yields the C++17-style initializer of the `if` or `switch` statement when it exists.
-
-## 0.2.1
-
-## 0.2.0
-
-### Breaking Changes
-
-* The signature of `allowImplicitRead` on `DataFlow::Configuration` and `TaintTracking::Configuration` has changed from `allowImplicitRead(DataFlow::Node node, DataFlow::Content c)` to `allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c)`.
-
-### Minor Analysis Improvements
-
-* More Windows pool allocation functions are now detected as `AllocationFunction`s.
-* The `semmle.code.cpp.commons.Buffer` library has been enhanced to handle array members of classes that do not specify a size.
-
 ## 0.1.0
 
 ### Breaking Changes

cpp/ql/lib/change-notes/2022-04-22-no-size-array.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The `semmle.code.cpp.commons.Buffer` library has been enhanced to handle array members of classes that do not specify a size.

cpp/ql/lib/change-notes/2022-04-25-allow-implicit-read-signature.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+The signature of `allowImplicitRead` on `DataFlow::Configuration` and `TaintTracking::Configuration` has changed from `allowImplicitRead(DataFlow::Node node, DataFlow::Content c)` to `allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c)`.

cpp/ql/lib/change-notes/2022-04-25-windows-pool-allocation-functions.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* More Windows pool allocation functions are now detected as `AllocationFunction`s.

cpp/ql/lib/change-notes/released/0.2.0.md

@@ -1,10 +0,0 @@
-## 0.2.0
-
-### Breaking Changes
-
-* The signature of `allowImplicitRead` on `DataFlow::Configuration` and `TaintTracking::Configuration` has changed from `allowImplicitRead(DataFlow::Node node, DataFlow::Content c)` to `allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c)`.
-
-### Minor Analysis Improvements
-
-* More Windows pool allocation functions are now detected as `AllocationFunction`s.
-* The `semmle.code.cpp.commons.Buffer` library has been enhanced to handle array members of classes that do not specify a size.

cpp/ql/lib/change-notes/released/0.2.1.md

@@ -1 +0,0 @@
-## 0.2.1

cpp/ql/lib/change-notes/released/0.2.2.md

@@ -1,9 +0,0 @@
-## 0.2.2
-
-### Deprecated APIs
-
- * The `AnalysedString` class in the `StringAnalysis` module has been replaced with `AnalyzedString`, to follow our style guide. The old name still exists as a deprecated alias.
-
-### New Features
-
-* A `getInitialization` predicate was added to the `ConstexprIfStmt`, `IfStmt`, and `SwitchStmt` classes that yields the C++17-style initializer of the `if` or `switch` statement when it exists.

cpp/ql/lib/change-notes/released/0.2.3.md

@@ -1,5 +0,0 @@
-## 0.2.3
-
-### New Features
-
-* An `isBraced` predicate was added to the `Initializer` class which holds when a C++ braced initializer was used in the initialization.

cpp/ql/lib/change-notes/released/0.3.0.md

@@ -1,9 +0,0 @@
-## 0.3.0
-
-### Deprecated APIs
-
-* The `BarrierGuard` class has been deprecated. Such barriers and sanitizers can now instead be created using the new `BarrierGuard` parameterized module.
-
-### Bug Fixes
-
-* `UserType.getADeclarationEntry()` now yields all forward declarations when the user type is a `class`, `struct`, or `union`.

cpp/ql/lib/change-notes/released/0.3.1.md

@@ -1,5 +0,0 @@
-## 0.3.1
-
-### Minor Analysis Improvements
-
-* `AnalysedExpr::isNullCheck` and `AnalysedExpr::isValidCheck` have been updated to handle variable accesses on the left-hand side of the C++ logical "and", and variable declarations in conditions.

cpp/ql/lib/change-notes/released/0.3.2.md

@@ -1,5 +0,0 @@
-## 0.3.2
-
-### Bug Fixes
-
-* Under certain circumstances a variable declaration that is not also a definition could be associated with a `Variable` that did not have the definition as a `VariableDeclarationEntry`. This is now fixed, and a unique `Variable` will exist that has both the declaration and the definition as a `VariableDeclarationEntry`.

cpp/ql/lib/change-notes/released/0.3.3.md

@@ -1,11 +0,0 @@
-## 0.3.3
-
-### New Features
-
-* Added a predicate `getValueConstant` to `AttributeArgument` that yields the argument value as an `Expr` when the value is a constant expression.
-* A new class predicate `MustFlowConfiguration::allowInterproceduralFlow` has been added to the `semmle.code.cpp.ir.dataflow.MustFlow` library. The new predicate can be overridden to disable interprocedural flow.
-* Added subclasses of `BuiltInOperations` for `__builtin_bit_cast`, `__builtin_shuffle`, `__has_unique_object_representations`, `__is_aggregate`, and `__is_assignable`.
-
-### Major Analysis Improvements
-
-* The IR dataflow library now includes flow through global variables. This enables new findings in many scenarios.

cpp/ql/lib/change-notes/released/0.3.4.md

@@ -1,15 +0,0 @@
-## 0.3.4
-
-### Deprecated APIs
-
-* Many classes/predicates/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
-  The old name still exists as a deprecated alias.
-
-### New Features
-
-* Added support for getting the link targets of global and namespace variables.
-* Added a `BlockAssignExpr` class, which models a `memcpy`-like operation used in compiler generated copy/move constructors and assignment operations.
-
-### Minor Analysis Improvements
-
-* All deprecated predicates/classes/modules that have been deprecated for over a year have been deleted.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.3.4
+lastReleaseVersion: 0.1.0

cpp/ql/lib/experimental/semmle/code/cpp/semantic/SemanticExprSpecific.qll

@@ -119,67 +119,27 @@ module SemanticExprConfig {
     result = block.getDisplayIndex()
   }
 
-  newtype TSsaVariable =
-    TSsaInstruction(IR::Instruction instr) { instr.hasMemoryResult() } or
-    TSsaOperand(IR::Operand op) { op.isDefinitionInexact() }
+  class SsaVariable instanceof IR::Instruction {
+    SsaVariable() { super.hasMemoryResult() }
 
-  class SsaVariable extends TSsaVariable {
-    string toString() { none() }
+    final string toString() { result = super.toString() }
 
-    Location getLocation() { none() }
-
-    IR::Instruction asInstruction() { none() }
-
-    IR::Operand asOperand() { none() }
+    final Location getLocation() { result = super.getLocation() }
   }
 
-  class SsaInstructionVariable extends SsaVariable, TSsaInstruction {
-    IR::Instruction instr;
+  predicate explicitUpdate(SsaVariable v, Expr sourceExpr) { v = sourceExpr }
 
-    SsaInstructionVariable() { this = TSsaInstruction(instr) }
+  predicate phi(SsaVariable v) { v instanceof IR::PhiInstruction }
 
-    final override string toString() { result = instr.toString() }
+  SsaVariable getAPhiInput(SsaVariable v) { result = v.(IR::PhiInstruction).getAnInput() }
 
-    final override Location getLocation() { result = instr.getLocation() }
-
-    final override IR::Instruction asInstruction() { result = instr }
-  }
-
-  class SsaOperand extends SsaVariable, TSsaOperand {
-    IR::Operand op;
-
-    SsaOperand() { this = TSsaOperand(op) }
-
-    final override string toString() { result = op.toString() }
-
-    final override Location getLocation() { result = op.getLocation() }
-
-    final override IR::Operand asOperand() { result = op }
-  }
-
-  predicate explicitUpdate(SsaVariable v, Expr sourceExpr) { v.asInstruction() = sourceExpr }
-
-  predicate phi(SsaVariable v) { v.asInstruction() instanceof IR::PhiInstruction }
-
-  SsaVariable getAPhiInput(SsaVariable v) {
-    exists(IR::PhiInstruction instr | v.asInstruction() = instr |
-      result.asInstruction() = instr.getAnInput()
-      or
-      result.asOperand() = instr.getAnInputOperand()
-    )
-  }
-
-  Expr getAUse(SsaVariable v) { result.(IR::LoadInstruction).getSourceValue() = v.asInstruction() }
+  Expr getAUse(SsaVariable v) { result.(IR::LoadInstruction).getSourceValue() = v }
 
   SemType getSsaVariableType(SsaVariable v) {
-    result = getSemanticType(v.asInstruction().getResultIRType())
+    result = getSemanticType(v.(IR::Instruction).getResultIRType())
   }
 
-  BasicBlock getSsaVariableBasicBlock(SsaVariable v) {
-    result = v.asInstruction().getBlock()
-    or
-    result = v.asOperand().getUse().getBlock()
-  }
+  BasicBlock getSsaVariableBasicBlock(SsaVariable v) { result = v.(IR::Instruction).getBlock() }
 
   private newtype TReadPosition =
     TReadPositionBlock(IR::IRBlock block) or
@@ -209,9 +169,7 @@ module SemanticExprConfig {
 
     final override predicate hasRead(SsaVariable v) {
       exists(IR::Operand operand |
-        operand.getDef() = v.asInstruction() and
-        not operand instanceof IR::PhiInputOperand and
-        operand.getUse().getBlock() = block
+        operand.getDef() = v and not operand instanceof IR::PhiInputOperand
       )
     }
   }
@@ -228,7 +186,7 @@ module SemanticExprConfig {
 
     final override predicate hasRead(SsaVariable v) {
       exists(IR::PhiInputOperand operand |
-        operand.getDef() = v.asInstruction() and
+        operand.getDef() = v and
         operand.getPredecessorBlock() = pred and
         operand.getUse().getBlock() = succ
       )
@@ -247,16 +205,17 @@ module SemanticExprConfig {
     exists(IR::PhiInputOperand operand |
       pos = TReadPositionPhiInputEdge(operand.getPredecessorBlock(), operand.getUse().getBlock())
     |
-      phi.asInstruction() = operand.getUse() and
-      (
-        input.asInstruction() = operand.getDef()
-        or
-        input.asOperand() = operand
-      )
+      phi = operand.getUse() and input = operand.getDef()
     )
   }
 
   class Bound instanceof IRBound::Bound {
+    Bound() {
+      this instanceof IRBound::ZeroBound
+      or
+      this.(IRBound::ValueNumberBound).getValueNumber().getAnInstruction() instanceof SsaVariable
+    }
+
     string toString() { result = super.toString() }
 
     final Location getLocation() { result = super.getLocation() }
@@ -269,13 +228,13 @@ module SemanticExprConfig {
 
     override string toString() {
       result =
-        min(SsaVariable v |
-          v.asInstruction() = bound.getValueNumber().getAnInstruction()
+        min(SsaVariable instr |
+          instr = bound.getValueNumber().getAnInstruction()
         |
-          v
+          instr
           order by
-            v.asInstruction().getBlock().getDisplayIndex(),
-            v.asInstruction().getDisplayIndexInBlock()
+            instr.(IR::Instruction).getBlock().getDisplayIndex(),
+            instr.(IR::Instruction).getDisplayIndexInBlock()
         ).toString()
     }
   }
@@ -283,7 +242,7 @@ module SemanticExprConfig {
   predicate zeroBound(Bound bound) { bound instanceof IRBound::ZeroBound }
 
   predicate ssaBound(Bound bound, SsaVariable v) {
-    v.asInstruction() = bound.(IRBound::ValueNumberBound).getValueNumber().getAnInstruction()
+    v = bound.(IRBound::ValueNumberBound).getValueNumber().getAnInstruction()
   }
 
   Expr getBoundExpr(Bound bound, int delta) {
@@ -292,20 +251,22 @@ module SemanticExprConfig {
 
   class Guard = IRGuards::IRGuardCondition;
 
-  predicate guard(Guard guard, BasicBlock block) { block = guard.getBlock() }
+  predicate guard(Guard guard, BasicBlock block) {
+    block = guard.(IRGuards::IRGuardCondition).getBlock()
+  }
 
   Expr getGuardAsExpr(Guard guard) { result = guard }
 
   predicate equalityGuard(Guard guard, Expr e1, Expr e2, boolean polarity) {
-    guard.comparesEq(e1.getAUse(), e2.getAUse(), 0, true, polarity)
+    guard.(IRGuards::IRGuardCondition).comparesEq(e1.getAUse(), e2.getAUse(), 0, true, polarity)
   }
 
   predicate guardDirectlyControlsBlock(Guard guard, BasicBlock controlled, boolean branch) {
-    guard.controls(controlled, branch)
+    guard.(IRGuards::IRGuardCondition).controls(controlled, branch)
   }
 
   predicate guardHasBranchEdge(Guard guard, BasicBlock bb1, BasicBlock bb2, boolean branch) {
-    guard.controlsEdge(bb1, bb2, branch)
+    guard.(IRGuards::IRGuardCondition).controlsEdge(bb1, bb2, branch)
   }
 
   Guard comparisonGuard(Expr e) { result = e }
@@ -323,13 +284,9 @@ SemBasicBlock getSemanticBasicBlock(IR::IRBlock block) { result = block }
 
 IR::IRBlock getCppBasicBlock(SemBasicBlock block) { block = result }
 
-SemSsaVariable getSemanticSsaVariable(IR::Instruction instr) {
-  result.(SemanticExprConfig::SsaVariable).asInstruction() = instr
-}
+SemSsaVariable getSemanticSsaVariable(IR::Instruction instr) { result = instr }
 
-IR::Instruction getCppSsaVariableInstruction(SemSsaVariable var) {
-  var.(SemanticExprConfig::SsaVariable).asInstruction() = result
-}
+IR::Instruction getCppSsaVariableInstruction(SemSsaVariable v) { v = result }
 
 SemBound getSemanticBound(IRBound::Bound bound) { result = bound }
 

cpp/ql/lib/experimental/semmle/code/cpp/semantic/analysis/ModulusAnalysis.qll

@@ -160,7 +160,6 @@ private predicate phiModulusInit(SemSsaPhiNode phi, SemBound b, int val, int mod
 /**
  * Holds if all inputs to `phi` numbered `1` to `rix` are equal to `b + val` modulo `mod`.
  */
-pragma[nomagic]
 private predicate phiModulusRankStep(SemSsaPhiNode phi, SemBound b, int val, int mod, int rix) {
   rix = 0 and
   phiModulusInit(phi, b, val, mod)
@@ -170,7 +169,7 @@ private predicate phiModulusRankStep(SemSsaPhiNode phi, SemBound b, int val, int
     val = remainder(v1, mod)
   |
     exists(int v2, int m2 |
-      rankedPhiInput(pragma[only_bind_out](phi), inp, edge, rix) and
+      rankedPhiInput(phi, inp, edge, rix) and
       phiModulusRankStep(phi, b, v1, m1, rix - 1) and
       ssaModulus(inp, edge, b, v2, m2) and
       mod = m1.gcd(m2).gcd(v1 - v2)

cpp/ql/lib/experimental/semmle/code/cpp/semantic/analysis/RangeAnalysis.qll

@@ -342,10 +342,7 @@ private class ConvertOrBoxExpr extends SemUnaryExpr {
  * A cast that can be ignored for the purpose of range analysis.
  */
 private class SafeCastExpr extends ConvertOrBoxExpr {
-  SafeCastExpr() {
-    conversionCannotOverflow(getTrackedType(pragma[only_bind_into](getOperand())),
-      getTrackedType(this))
-  }
+  SafeCastExpr() { conversionCannotOverflow(getTrackedType(getOperand()), getTrackedType(this)) }
 }
 
 /**

cpp/ql/lib/experimental/semmle/code/cpp/semantic/analysis/SignAnalysisCommon.qll

@@ -189,12 +189,9 @@ private class BinarySignExpr extends FlowSignExpr {
   BinarySignExpr() { binary = this }
 
   override Sign getSignRestriction() {
-    exists(SemExpr left, SemExpr right |
-      binaryExprOperands(binary, left, right) and
-      result =
-        semExprSign(pragma[only_bind_out](left))
-            .applyBinaryOp(semExprSign(pragma[only_bind_out](right)), binary.getOpcode())
-    )
+    result =
+      semExprSign(binary.getLeftOperand())
+          .applyBinaryOp(semExprSign(binary.getRightOperand()), binary.getOpcode())
     or
     exists(SemDivExpr div | div = binary |
       result = semExprSign(div.getLeftOperand()) and
@@ -204,11 +201,6 @@ private class BinarySignExpr extends FlowSignExpr {
   }
 }
 
-pragma[nomagic]
-private predicate binaryExprOperands(SemBinaryExpr binary, SemExpr left, SemExpr right) {
-  binary.getLeftOperand() = left and binary.getRightOperand() = right
-}
-
 /**
  * A `Convert`, `Box`, or `Unbox` expression.
  */
@@ -229,7 +221,7 @@ private class UnarySignExpr extends FlowSignExpr {
   UnarySignExpr() { unary = this and not this instanceof SemCastExpr }
 
   override Sign getSignRestriction() {
-    result = semExprSign(pragma[only_bind_out](unary.getOperand())).applyUnaryOp(unary.getOpcode())
+    result = semExprSign(unary.getOperand()).applyUnaryOp(unary.getOpcode())
   }
 }
 

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.3.5-dev
+version: 0.1.1-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/Class.qll

@@ -404,10 +404,7 @@ class Class extends UserType {
    * compiled for. For this reason, the `is_pod_class` predicate is
    * generated by the extractor.
    */
-  predicate isPod() { is_pod_class(underlyingElement(this)) }
-
-  /** DEPRECATED: Alias for isPod */
-  deprecated predicate isPOD() { this.isPod() }
+  predicate isPOD() { is_pod_class(underlyingElement(this)) }
 
   /**
    * Holds if this class, struct or union is a standard-layout class

cpp/ql/lib/semmle/code/cpp/Element.qll

@@ -6,7 +6,6 @@
 import semmle.code.cpp.Location
 private import semmle.code.cpp.Enclosing
 private import semmle.code.cpp.internal.ResolveClass
-private import semmle.code.cpp.internal.ResolveGlobalVariable
 
 /**
  * Get the `Element` that represents this `@element`.
@@ -29,12 +28,9 @@ Element mkElement(@element e) { unresolveElement(result) = e }
 pragma[inline]
 @element unresolveElement(Element e) {
   not result instanceof @usertype and
-  not result instanceof @variable and
   result = e
   or
   e = resolveClass(result)
-  or
-  e = resolveGlobalVariable(result)
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/Field.qll

@@ -31,7 +31,7 @@ class Field extends MemberVariable {
   int getByteOffset() { fieldoffsets(underlyingElement(this), result, _) }
 
   /**
-   * Gets the byte offset within `mostDerivedClass` of each occurrence of this
+   * Gets the byte offset within `mostDerivedClass` of each occurence of this
    * field within `mostDerivedClass` itself or a base class subobject of
    * `mostDerivedClass`.
    * Note that for fields of virtual base classes, and non-virtual base classes

cpp/ql/lib/semmle/code/cpp/File.qll

@@ -218,6 +218,8 @@ class Folder extends Container, @folder {
 class File extends Container, @file {
   override string getAbsolutePath() { files(underlyingElement(this), result) }
 
+  override string toString() { result = Container.super.toString() }
+
   override string getAPrimaryQlClass() { result = "File" }
 
   override Location getLocation() {

cpp/ql/lib/semmle/code/cpp/Function.qll

@@ -38,8 +38,8 @@ class Function extends Declaration, ControlFlowNode, AccessHolder, @function {
    * int z = min(5, 7);
    * ```
    * The full signature of the function called on the last line would be
-   * `min<int>(int, int) -> int`, and the full signature of the uninstantiated
-   * template on the first line would be `min<T>(T, T) -> T`.
+   * "min<int>(int, int) -> int", and the full signature of the uninstantiated
+   * template on the first line would be "min<T>(T, T) -> T".
    */
   string getFullSignature() {
     exists(string name, string templateArgs, string args |

cpp/ql/lib/semmle/code/cpp/Initializer.qll

@@ -51,7 +51,4 @@ class Initializer extends ControlFlowNode, @initialiser {
   override Function getControlFlowScope() { result = this.getExpr().getEnclosingFunction() }
 
   override Stmt getEnclosingStmt() { result = this.getExpr().getEnclosingStmt() }
-
-  /** Holds if the initializer used the C++ braced initializer notation. */
-  predicate isBraced() { braced_initialisers(underlyingElement(this)) }
 }

cpp/ql/lib/semmle/code/cpp/Linkage.qll

@@ -41,15 +41,6 @@ class LinkTarget extends @link_target {
    * translation units which contributed to this link target.
    */
   Class getAClass() { link_parent(unresolveElement(result), this) }
-
-  /**
-   * Gets a global or namespace variable which was compiled into this
-   * link target, or had its declaration included by one of the translation
-   * units which contributed to this link target.
-   */
-  GlobalOrNamespaceVariable getAGlobalOrNamespaceVariable() {
-    link_parent(unresolveElement(result), this)
-  }
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/PODType03.qll

@@ -79,17 +79,17 @@ predicate isAggregateType03(Type t) {
  *   user-defined copy assignment operator and no user-defined destructor.
  *   A POD class is a class that is either a POD-struct or a POD-union.
  */
-predicate isPodClass03(Class c) {
+predicate isPODClass03(Class c) {
   isAggregateClass03(c) and
   not exists(Variable v |
     v.getDeclaringType() = c and
     not v.isStatic()
   |
-    not isPodType03(v.getType())
+    not isPODType03(v.getType())
     or
     exists(ArrayType at |
       at = v.getType() and
-      not isPodType03(at.getBaseType())
+      not isPODType03(at.getBaseType())
     )
     or
     v.getType() instanceof ReferenceType
@@ -104,9 +104,6 @@ predicate isPodClass03(Class c) {
   )
 }
 
-/** DEPRECATED: Alias for isPodClass03 */
-deprecated predicate isPODClass03 = isPodClass03/1;
-
 /**
  * Holds if `t` is a POD type, according to the rules specified in
  * C++03 3.9(10):
@@ -115,17 +112,14 @@ deprecated predicate isPODClass03 = isPodClass03/1;
  *   such types and cv-qualified versions of these types (3.9.3) are
  *   collectively called POD types.
  */
-predicate isPodType03(Type t) {
+predicate isPODType03(Type t) {
   exists(Type ut | ut = t.getUnderlyingType() |
     isScalarType03(ut)
     or
-    isPodClass03(ut)
+    isPODClass03(ut)
     or
-    exists(ArrayType at | at = ut and isPodType03(at.getBaseType()))
+    exists(ArrayType at | at = ut and isPODType03(at.getBaseType()))
     or
-    isPodType03(ut.(SpecifiedType).getUnspecifiedType())
+    isPODType03(ut.(SpecifiedType).getUnspecifiedType())
   )
 }
-
-/** DEPRECATED: Alias for isPodType03 */
-deprecated predicate isPODType03 = isPodType03/1;

cpp/ql/lib/semmle/code/cpp/PrintAST.qll

@@ -663,24 +663,18 @@ private predicate namedStmtChildPredicates(Locatable s, Element e, string pred)
     or
     s.(ComputedGotoStmt).getExpr() = e and pred = "getExpr()"
     or
-    s.(ConstexprIfStmt).getInitialization() = e and pred = "getInitialization()"
-    or
     s.(ConstexprIfStmt).getCondition() = e and pred = "getCondition()"
     or
     s.(ConstexprIfStmt).getThen() = e and pred = "getThen()"
     or
     s.(ConstexprIfStmt).getElse() = e and pred = "getElse()"
     or
-    s.(IfStmt).getInitialization() = e and pred = "getInitialization()"
-    or
     s.(IfStmt).getCondition() = e and pred = "getCondition()"
     or
     s.(IfStmt).getThen() = e and pred = "getThen()"
     or
     s.(IfStmt).getElse() = e and pred = "getElse()"
     or
-    s.(SwitchStmt).getInitialization() = e and pred = "getInitialization()"
-    or
     s.(SwitchStmt).getExpr() = e and pred = "getExpr()"
     or
     s.(SwitchStmt).getStmt() = e and pred = "getStmt()"

cpp/ql/lib/semmle/code/cpp/Specifier.qll

@@ -12,7 +12,7 @@ private import semmle.code.cpp.internal.ResolveClass
 class Specifier extends Element, @specifier {
   /** Gets a dummy location for the specifier. */
   override Location getLocation() {
-    exists(this) and
+    suppressUnusedThis(this) and
     result instanceof UnknownDefaultLocation
   }
 
@@ -256,13 +256,9 @@ class AttributeArgument extends Element, @attribute_arg {
 
   /**
    * Gets the text for the value of this argument, if its value is
-   * a constant or a token.
+   * a string or a number.
    */
-  string getValueText() {
-    if underlyingElement(this) instanceof @attribute_arg_constant_expr
-    then result = this.getValueConstant().getValue()
-    else attribute_arg_value(underlyingElement(this), result)
-  }
+  string getValueText() { attribute_arg_value(underlyingElement(this), result) }
 
   /**
    * Gets the value of this argument, if its value is integral.
@@ -274,13 +270,6 @@ class AttributeArgument extends Element, @attribute_arg {
    */
   Type getValueType() { attribute_arg_type(underlyingElement(this), unresolveElement(result)) }
 
-  /**
-   * Gets the value of this argument, if its value is a constant.
-   */
-  Expr getValueConstant() {
-    attribute_arg_constant(underlyingElement(this), unresolveElement(result))
-  }
-
   /**
    * Gets the attribute to which this is an argument.
    */
@@ -305,12 +294,11 @@ class AttributeArgument extends Element, @attribute_arg {
         (
           if underlyingElement(this) instanceof @attribute_arg_type
           then tail = this.getValueType().getName()
-          else
-            if underlyingElement(this) instanceof @attribute_arg_constant_expr
-            then tail = this.getValueConstant().toString()
-            else tail = this.getValueText()
+          else tail = this.getValueText()
         ) and
         result = prefix + tail
       )
   }
 }
+
+private predicate suppressUnusedThis(Specifier s) { any() }

cpp/ql/lib/semmle/code/cpp/UserType.qll

@@ -48,8 +48,8 @@ class UserType extends Type, Declaration, NameQualifyingElement, AccessHolder, @
   }
 
   override TypeDeclarationEntry getADeclarationEntry() {
-    if type_decls(_, unresolveElement(this), _)
-    then type_decls(underlyingElement(result), unresolveElement(this), _)
+    if type_decls(_, underlyingElement(this), _)
+    then type_decls(unresolveElement(result), underlyingElement(this), _)
     else exists(Class t | this.(Class).isConstructedFrom(t) and result = t.getADeclarationEntry())
   }
 

cpp/ql/lib/semmle/code/cpp/Variable.qll

@@ -6,7 +6,6 @@ import semmle.code.cpp.Element
 import semmle.code.cpp.exprs.Access
 import semmle.code.cpp.Initializer
 private import semmle.code.cpp.internal.ResolveClass
-private import semmle.code.cpp.internal.ResolveGlobalVariable
 
 /**
  * A C/C++ variable. For example, in the following code there are four
@@ -33,8 +32,6 @@ private import semmle.code.cpp.internal.ResolveGlobalVariable
  * can have multiple declarations.
  */
 class Variable extends Declaration, @variable {
-  Variable() { isVariable(underlyingElement(this)) }
-
   override string getAPrimaryQlClass() { result = "Variable" }
 
   /** Gets the initializer of this variable, if any. */
@@ -398,8 +395,6 @@ class LocalVariable extends LocalScopeVariable, @localvariable {
     exists(DeclStmt s | s.getADeclaration() = this and s.getEnclosingFunction() = result)
     or
     exists(ConditionDeclExpr e | e.getVariable() = this and e.getEnclosingFunction() = result)
-    or
-    orphaned_variables(underlyingElement(this), unresolveElement(result))
   }
 }
 
@@ -473,9 +468,6 @@ class GlobalOrNamespaceVariable extends Variable, @globalvariable {
   override Type getType() { globalvariables(underlyingElement(this), unresolveElement(result), _) }
 
   override Element getEnclosingElement() { none() }
-
-  /** Gets a link target which compiled or referenced this global or namespace variable. */
-  LinkTarget getALinkTarget() { this = result.getAGlobalOrNamespaceVariable() }
 }
 
 /**