Merge branch 'github:main' into amammad-java-bombs

.bazelrc

@@ -1,3 +1,24 @@
-build --repo_env=CC=clang --repo_env=CXX=clang++ --cxxopt="-std=c++17"
+common --enable_platform_specific_config
+common --enable_bzlmod
+# because we use --override_module with `%workspace%`, the lock file is not stable
+common --lockfile_mode=off
+
+# when building from this repository in isolation, the internal repository will not be found at ..
+# where `MODULE.bazel` looks for it. The following will get us past the module loading phase, so
+# that we can build things that do not rely on that
+common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
+
+build --repo_env=CC=clang --repo_env=CXX=clang++
+
+build:linux --cxxopt=-std=c++20
+build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
+build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
+
+# this requires developer mode, but is required to have pack installer functioning
+startup --windows_enable_symlinks
+common --enable_runfiles
+
+common --registry=file:///%workspace%/misc/bazel/registry
+common --registry=https://bcr.bazel.build
 
 try-import %workspace%/local.bazelrc

.bazelrc.internal

@@ -0,0 +1,4 @@
+# this file should contain bazel settings required to build things from `semmle-code`
+
+common --registry=file:///%workspace%/ql/misc/bazel/registry
+common --registry=https://bcr.bazel.build

.bazelversion

@@ -1 +1 @@
-6.1.2
+7.1.0

.clang-format

@@ -0,0 +1 @@
+DisableFormat: true

.gitattributes

@@ -67,7 +67,9 @@ go/extractor/opencsv/CSVReader.java -text
 # for those testing dbscheme files.
 */ql/lib/upgrades/initial/*.dbscheme -text
 
-# Generated test files - these are synced from the standard JavaScript libraries using
-# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge
+# Auto-generated modeling for Python
+python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
+
+# auto-generated bazel lock file
+ruby/extractor/cargo-bazel-lock.json linguist-generated=true
+ruby/extractor/cargo-bazel-lock.json -merge

.github/dependabot.yml

@@ -17,3 +17,26 @@ updates:
     ignore:
       - dependency-name: '*'
         update-types: ['version-update:semver-patch', 'version-update:semver-minor']
+
+  - package-ecosystem: "gomod"
+    directory: "go/extractor"
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-name: "golang.org/x/mod"
+      - dependency-name: "golang.org/x/tools"
+    groups:
+      extractor-dependencies:
+        patterns:
+          - "golang.org/x/*"
+    reviewers:
+      - "github/codeql-go"
+
+  - package-ecosystem: "gomod"
+    directory: "go/ql/test"
+    schedule:
+      interval: "monthly"
+    ignore:
+      - dependency-name: "*"
+    reviewers:
+      - "github/codeql-go"

.github/labeler.yml

@@ -15,12 +15,12 @@ Java:
   - change-notes/**/*java.*
 
 JS:
-  - any: [ 'javascript/**/*', '!javascript/ql/experimental/adaptivethreatmodeling/**/*' ]
+  - any: [ 'javascript/**/*' ]
   - change-notes/**/*javascript*
 
 Kotlin:
   - java/kotlin-extractor/**/*
-  - java/ql/test/kotlin/**/*
+  - java/ql/test-kotlin*/**/*
 
 Python:
   - python/**/*
@@ -45,11 +45,4 @@ documentation:
 
 # Since these are all shared files that need to be synced, just pick _one_ copy of each.
 "DataFlow Library":
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"
-
-"ATM":
-  - javascript/ql/experimental/adaptivethreatmodeling/**/*
+  - "shared/dataflow/**/*"

.github/workflows/buildifier.yml

@@ -0,0 +1,28 @@
+name: Check bazel formatting
+
+on:
+  pull_request:
+    paths:
+      - "**.bazel"
+      - "**.bzl"
+    branches:
+      - main
+      - "rc/*"
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Check bazel formatting
+        uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
+        with:
+          extra_args: >
+            buildifier --all-files 2>&1 ||
+            (
+              echo -e "In order to format all bazel files, please run:\n  bazel run //misc/bazel:buildifier"; exit 1
+            )

.github/workflows/check-change-note.yml

@@ -1,5 +1,8 @@
 name: Check change note
 
+permissions:
+  pull-requests: read
+
 on:
   pull_request_target:
     types: [labeled, unlabeled, opened, synchronize, reopened, ready_for_review]
@@ -9,26 +12,42 @@ on:
       - "*/ql/lib/**/*.ql"
       - "*/ql/lib/**/*.qll"
       - "*/ql/lib/**/*.yml"
+      - "shared/**/*.ql"
+      - "shared/**/*.qll"
       - "!**/experimental/**"
       - "!ql/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:
   check-change-note:
+    env: 
+      REPO: ${{ github.repository }}
+      PULL_REQUEST_NUMBER: ${{ github.event.number }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     runs-on: ubuntu-latest
     steps:
+
       - name: Fail if no change note found. To fix, either add one, or add the `no-change-note-required` label.
         if: |
           github.event.pull_request.draft == false &&
           !contains(github.event.pull_request.labels.*.name, 'no-change-note-required')
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' |
-          grep true -c
+          change_note_files=$(gh api "repos/$REPO/pulls/$PULL_REQUEST_NUMBER/files" --paginate --jq '.[].filename | select(test("/change-notes/.*[.]md$"))')
+          
+          if [ -z "$change_note_files" ]; then
+            echo "No change note found. Either add one, or add the 'no-change-note-required' label."
+            exit 1
+          fi
+
+          echo "Change notes found:"
+          echo "$change_note_files"
+
       - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md', 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text, or released/x.y.z.md for released change-notes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$") or test("/change-notes/released/[0-9]*[.][0-9]*[.][0-9]*[.]md$"))' |
-          grep true -c
+          bad_change_note_file_names=$(gh api "repos/$REPO/pulls/$PULL_REQUEST_NUMBER/files" --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))][] | select((test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$") or test("/change-notes/released/[0-9]*[.][0-9]*[.][0-9]*[.]md$")) | not)')
+
+          if [ -n "$bad_change_note_file_names" ]; then
+            echo "The following change note file names are invalid:"
+            echo "$bad_change_note_file_names"
+            exit 1
+          fi

.github/workflows/check-implicit-this.yml

@@ -0,0 +1,32 @@
+name: "Check implicit this warnings"
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "**qlpack.yml"
+    branches:
+      - main
+      - "rc/*"
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check that implicit this warnings is enabled for all packs
+        shell: bash
+        run: |
+          EXIT_CODE=0
+          packs="$(find . -iname 'qlpack.yml')"
+          for pack_file in ${packs}; do
+            option="$(yq '.warnOnImplicitThis' ${pack_file})"
+            if [ "${option}" != "true" ]; then
+              echo "::error file=${pack_file}::warnOnImplicitThis property must be set to 'true' for pack ${pack_file}"
+              EXIT_CODE=1
+            fi
+          done
+          exit "${EXIT_CODE}"

.github/workflows/check-qldoc.yml

@@ -10,12 +10,15 @@ on:
       - main
       - "rc/*"
 
+permissions:
+  contents: read
+
 jobs:
   qldoc:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 2
 

.github/workflows/check-query-ids.yml

@@ -11,11 +11,14 @@ on:
       - "rc/*"
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   check:
     name: Check query IDs
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Check for duplicate query IDs
         run: python3 misc/scripts/check-query-ids.py

.github/workflows/close-stale.yml

@@ -5,6 +5,9 @@ on:
   schedule:
     - cron: "30 1 * * *"
 
+permissions:
+  issues: write
+
 jobs:
   stale:
     if: github.repository == 'github/codeql'
@@ -12,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/stale@v8
+    - uses: actions/stale@v9
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Comment or remove the `Stale` label in order to avoid having this issue closed in 7 days.'

.github/workflows/codeql-analysis.yml

@@ -28,12 +28,12 @@ jobs:
 
     steps:
     - name: Setup dotnet
-      uses: actions/setup-dotnet@v3
+      uses: actions/setup-dotnet@v4
       with:
-        dotnet-version: 7.0.102
+        dotnet-version: 8.0.101
 
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/compile-queries.yml

@@ -8,12 +8,16 @@ on:
       - "codeql-cli-*"
   pull_request:
 
+permissions:
+  contents: read
+
 jobs:
   compile-queries:
+    if: github.repository_owner == 'github'
     runs-on: ubuntu-latest-xl
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
         with:
@@ -24,14 +28,14 @@ jobs:
         with: 
           key: all-queries
       - name: check formatting
-        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 -n 3000 -P 10 codeql query format -q --check-only
+        run: find shared */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 -n 3000 -P 10 codeql query format -q --check-only
       - name: compile queries - check-only
         # run with --check-only if running in a PR (github.sha != main)
         if : ${{ github.event_name == 'pull_request' }}
         shell: bash
-        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" --compilation-cache-size=500
       - name: compile queries - full
         # do full compile if running on main - this populates the cache
         if : ${{ github.event_name != 'pull_request' }}
         shell: bash
-        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" --compilation-cache-size=500

.github/workflows/cpp-swift-analysis.yml

@@ -0,0 +1,55 @@
+name: "Code scanning - C++"
+
+on:
+  push:
+    branches:
+      - main
+      - 'rc/*'
+  pull_request:
+    branches:
+      - main
+      - 'rc/*'
+    paths:
+      - 'swift/**'
+      - '.github/codeql/**'
+      - '.github/workflows/cpp-swift-analysis.yml'
+  schedule:
+    - cron: '0 9 * * 1'
+
+jobs:
+  CodeQL-Build:
+
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      security-events: write
+      pull-requests: read
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@main
+      # Override language selection by uncommenting this and choosing your languages
+      with:
+        languages: cpp
+        config-file: ./.github/codeql/codeql-config.yml
+  
+    - name: "[Ubuntu] Remove GCC 13 from runner image"
+      shell: bash
+      run: |
+        sudo rm -f /etc/apt/sources.list.d/ubuntu-toolchain-r-ubuntu-test-jammy.list
+        sudo apt-get update
+        sudo apt-get install -y --allow-downgrades libc6=2.35-* libc6-dev=2.35-* libstdc++6=12.3.0-* libgcc-s1=12.3.0-*
+
+    - name: "Build Swift extractor using Bazel"
+      run: |
+        bazel clean --expunge
+        bazel run //swift:create-extractor-pack --nouse_action_cache --noremote_accept_cached --noremote_upload_local_results --spawn_strategy=local --features=-layering_check
+        bazel shutdown
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@main

.github/workflows/csharp-qltest.yml

@@ -25,11 +25,14 @@ defaults:
   run:
     working-directory: csharp
 
+permissions:
+  contents: read
+
 jobs:
   qlupgrade:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/fetch-codeql
       - name: Check DB upgrade scripts
         run: |
@@ -46,14 +49,14 @@ jobs:
            xargs codeql execute upgrades testdb
           diff -q testdb/semmlecode.csharp.dbscheme downgrades/initial/semmlecode.csharp.dbscheme
   qltest:
+    if: github.repository_owner == 'github'
     runs-on: ubuntu-latest-xl
     strategy:
       fail-fast: false
       matrix:
         slice: ["1/2", "2/2"]
     steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
+      - uses: actions/checkout@v4
       - uses: ./csharp/actions/create-extractor-pack
       - name: Cache compilation cache
         id: query-cache
@@ -62,25 +65,41 @@ jobs:
           key: csharp-qltest-${{ matrix.slice }}
       - name: Run QL tests
         run: |
-          CODEQL_PATH=$(gh codeql version --format=json | jq -r .unpackedLocation)
-          # The legacy ASP extractor is not in this repo, so take the one from the nightly build
-          mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
-          # Safe guard against using the bundled extractor
-          rm -rf "$CODEQL_PATH/csharp"
-          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}
   unit-tests:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-2019]
+    runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup dotnet
-        uses: actions/setup-dotnet@v3
+        uses: actions/setup-dotnet@v4
         with:
-          dotnet-version: 7.0.102
+          dotnet-version: 8.0.101
       - name: Extractor unit tests
         run: |
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/extractor/Semmle.Util.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/extractor/Semmle.Extraction.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"
+          dotnet test -p:RuntimeFrameworkVersion=8.0.1 extractor/Semmle.Util.Tests
+          dotnet test -p:RuntimeFrameworkVersion=8.0.1 extractor/Semmle.Extraction.Tests
+          dotnet test -p:RuntimeFrameworkVersion=8.0.1 autobuilder/Semmle.Autobuild.CSharp.Tests
+          dotnet test -p:RuntimeFrameworkVersion=8.0.1 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"
+        shell: bash
+  stubgentest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./csharp/actions/create-extractor-pack
+      - name: Run stub generator tests
+        run: |
+          # Generate (Asp)NetCore stubs
+          STUBS_PATH=stubs_output
+          python3 scripts/stubs/make_stubs_nuget.py webapp Swashbuckle.AspNetCore.Swagger 6.5.0 "$STUBS_PATH"
+          rm -rf ql/test/resources/stubs/_frameworks
+          # Update existing stubs in the repo with the freshly generated ones
+          mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/
+          git status
+          codeql test run --threads=0 --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/csv-coverage-metrics.yml

@@ -14,12 +14,16 @@ on:
       - ".github/workflows/csv-coverage-metrics.yml"
       - ".github/actions/fetch-codeql/action.yml"
 
+permissions:
+  contents: read
+  security-events: write
+
 jobs:
   publish-java:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
       - name: Create empty database
@@ -47,7 +51,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
       - name: Create empty database

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -19,6 +19,10 @@ on:
       - main
       - "rc/*"
 
+permissions:
+  contents: read
+  pull-requests: read
+
 jobs:
   generate:
     name: Generate framework coverage artifacts
@@ -31,11 +35,11 @@ jobs:
           GITHUB_CONTEXT: ${{ toJSON(github.event) }}
         run: echo "$GITHUB_CONTEXT"
       - name: Clone self (github/codeql) - MERGE
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: merge
       - name: Clone self (github/codeql) - BASE
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 2
           path: base
@@ -89,9 +93,32 @@ jobs:
       - name: Save PR number
         run: |
           mkdir -p pr
-          echo ${{ github.event.pull_request.number }} > pr/NR
+          echo ${PR_NUMBER} > pr/NR
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
       - name: Upload PR number
         uses: actions/upload-artifact@v3
         with:
           name: pr
           path: pr/
+      - name: Save comment ID (if it exists)
+        run: |
+          # Find the latest comment starting with COMMENT_PREFIX
+          COMMENT_PREFIX=":warning: The head of this PR and the base branch were compared for differences in the framework coverage reports."
+          COMMENT_ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate | jq --arg prefix "${COMMENT_PREFIX}" 'map(select(.body|startswith($prefix)) | .id) | max // empty')
+          if [[ -z ${COMMENT_ID} ]]
+          then
+            echo "Comment not found. Not uploading 'comment/ID' artifact."
+          else
+            mkdir -p comment
+            echo ${COMMENT_ID} > comment/ID
+          fi
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+      - name: Upload comment ID (if it exists)
+        uses: actions/upload-artifact@v3
+        with:
+          name: comment
+          path: comment/
+          if-no-files-found: ignore

.github/workflows/csv-coverage-pr-comment.yml

@@ -6,6 +6,10 @@ on:
     types:
       - completed
 
+permissions:
+  contents: read
+  pull-requests: write
+
 jobs:
   check:
     name: Check framework coverage differences and comment
@@ -20,7 +24,7 @@ jobs:
         GITHUB_CONTEXT: ${{ toJSON(github.event) }}
       run: echo "$GITHUB_CONTEXT"
     - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Set up Python 3.8
       uses: actions/setup-python@v4
       with:

.github/workflows/csv-coverage-timeseries.yml

@@ -3,17 +3,20 @@ name: Build framework coverage timeseries reports
 on:
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ubuntu-latest
 
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: script
       - name: Clone self (github/codeql) for analysis
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: codeqlModels
           fetch-depth: 0

.github/workflows/csv-coverage-update.yml

@@ -5,6 +5,10 @@ on:
   schedule:
     - cron: "0 0 * * *"
 
+permissions:
+  contents: write
+  pull-requests: write
+
 jobs:
   update:
     name: Update framework coverage report
@@ -17,7 +21,7 @@ jobs:
           GITHUB_CONTEXT: ${{ toJSON(github.event) }}
         run: echo "$GITHUB_CONTEXT"
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: ql
           fetch-depth: 0

.github/workflows/csv-coverage.yml

@@ -7,17 +7,20 @@ on:
         description: "github/codeql repo SHA used for looking up the CSV models"
         required: false
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ubuntu-latest
 
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: script
       - name: Clone self (github/codeql) for analysis
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: codeqlModels
           ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}

.github/workflows/fast-forward.yml

@@ -7,13 +7,14 @@ name: Fast-forward tracking branch for selected CodeQL version
 on:
   workflow_dispatch:
 
+permissions:
+  contents: write
+
 jobs:
   fast-forward:
     name: Fast-forward tracking branch for selected CodeQL version
     runs-on: ubuntu-latest
     if: github.repository == 'github/codeql'
-    permissions:
-      contents: write
     env:
       BRANCH_NAME: 'lgtm.com'
     steps:
@@ -25,7 +26,7 @@ jobs:
           exit 1
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Git config
         shell: bash

.github/workflows/go-tests-other-os.yml

@@ -7,19 +7,26 @@ on:
       - .github/workflows/go-tests-other-os.yml
       - .github/actions/**
       - codeql-workspace.yml
+env:
+  GO_VERSION: '~1.22.0'
+
+permissions:
+  contents: read
+
 jobs:
   test-mac:
     name: Test MacOS
     runs-on: macos-latest
     steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v4
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
         id: go
 
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up CodeQL CLI
         uses: ./.github/actions/fetch-codeql
@@ -44,17 +51,19 @@ jobs:
           make test cache="${{ steps.query-cache.outputs.cache-dir }}"
 
   test-win:
+    if: github.repository_owner == 'github'
     name: Test Windows
     runs-on: windows-latest-xl
     steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v4
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
         id: go
 
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up CodeQL CLI
         uses: ./.github/actions/fetch-codeql

.github/workflows/go-tests.yml

@@ -15,19 +15,28 @@ on:
       - .github/workflows/go-tests.yml
       - .github/actions/**
       - codeql-workspace.yml
+
+env:
+  GO_VERSION: '~1.22.0'
+
+permissions:
+  contents: read
+
 jobs:
   test-linux:
+    if: github.repository_owner == 'github'
     name: Test Linux (Ubuntu)
     runs-on: ubuntu-latest-xl
     steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v4
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
         id: go
 
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up CodeQL CLI
         uses: ./.github/actions/fetch-codeql

.github/workflows/js-ml-tests.yml

@@ -1,65 +0,0 @@
-name: JS ML-powered queries tests
-
-on:
-  push:
-    paths:
-      - "javascript/ql/experimental/adaptivethreatmodeling/**"
-      - .github/workflows/js-ml-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "javascript/ql/experimental/adaptivethreatmodeling/**"
-      - .github/workflows/js-ml-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-  workflow_dispatch:
-
-defaults:
-  run:
-    working-directory: javascript/ql/experimental/adaptivethreatmodeling
-
-jobs:
-  qltest:
-    name: Test QL
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: ./.github/actions/fetch-codeql
-
-      - name: Install pack dependencies
-        run: |
-          for pack in modelbuilding src test; do
-            codeql pack install --mode verify -- "${pack}"
-          done
-      
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: js-ml-test
-
-      - name: Check QL compilation
-        run: |
-          codeql query compile \
-            --check-only \
-            --ram 50000 \
-            --additional-packs "${{ github.workspace }}" \
-            --threads=0 \
-            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
-            -- \
-            lib modelbuilding src
-
-      - name: Run QL tests
-        run: |
-          codeql test run \
-            --threads=0 \
-            --ram 50000 \
-            --additional-packs "${{ github.workspace }}" \
-            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
-            -- \
-            test

.github/workflows/labeler.yml

@@ -2,11 +2,12 @@ name: "Pull Request Labeler"
 on:
 - pull_request_target
 
+permissions:
+  contents: read
+  pull-requests: write
+
 jobs:
   triage:
-    permissions:
-      contents: read
-      pull-requests: write
     runs-on: ubuntu-latest
     steps:
     - uses: actions/labeler@v4

.github/workflows/mad_modelDiff.yml

@@ -12,6 +12,7 @@ on:
       - main
     paths:
       - "java/ql/src/utils/modelgenerator/**/*.*"
+      - "misc/scripts/models-as-data/*.*"
       - ".github/workflows/mad_modelDiff.yml"
 
 permissions:
@@ -27,12 +28,12 @@ jobs:
         slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
     steps:
       - name: Clone github/codeql from PR
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         if: github.event.pull_request
         with:
           path: codeql-pr
       - name: Clone github/codeql from main
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: codeql-main
           ref: main
@@ -61,8 +62,9 @@ jobs:
             DATABASE=$2
             cd codeql-$QL_VARIANT
             SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE ${SHORTNAME}.temp.model.yml
-            mv java/ql/lib/ext/generated/${SHORTNAME}.temp.model.yml $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.model.yml
+            python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $SHORTNAME/$QL_VARIANT
+            mkdir -p $MODELS/$SHORTNAME
+            mv java/ql/lib/ext/generated/$SHORTNAME/$QL_VARIANT $MODELS/$SHORTNAME
             cd ..
           }
 
@@ -85,16 +87,16 @@ jobs:
           set -x
           MODELS=`pwd`/tmp-models
           ls -1 tmp-models/
-          for m in $MODELS/*_main.model.yml ; do
+          for m in $MODELS/*/main/*.model.yml ; do
             t="${m/main/"pr"}"
             basename=`basename $m`
-            name="diff_${basename/_main.model.yml/""}"
+            name="diff_${basename/.model.yml/""}"
             (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
           done
       - uses: actions/upload-artifact@v3
         with:
           name: models
-          path: tmp-models/*.model.yml
+          path: tmp-models/**/**/*.model.yml
           retention-days: 20
       - uses: actions/upload-artifact@v3
         with:

.github/workflows/mad_regenerate-models.yml

@@ -11,6 +11,9 @@ on:
       - ".github/workflows/mad_regenerate-models.yml"
       - ".github/actions/fetch-codeql/action.yml"
 
+permissions:
+  contents: read
+
 jobs:
   regenerate-models:
     runs-on: ubuntu-latest
@@ -27,11 +30,11 @@ jobs:
             ref: "placeholder"
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup CodeQL binaries
         uses: ./.github/actions/fetch-codeql
       - name: Clone repositories
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: repos/${{ matrix.ref }}
           ref: ${{ matrix.ref }}

.github/workflows/qhelp-pr-preview.yml

@@ -43,7 +43,7 @@ jobs:
           if-no-files-found: error
           retention-days: 1
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 2
           persist-credentials: false
@@ -77,7 +77,7 @@ jobs:
           done < "${RUNNER_TEMP}/paths.txt" >> comment_body.txt
           exit "${EXIT_CODE}"
 
-      - if: always()
+      - if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v3
         with:
           name: comment

.github/workflows/ql-for-ql-build.yml

@@ -9,17 +9,22 @@ on:
 env:
   CARGO_TERM_COLOR: always
 
+permissions:
+  contents: read
+  security-events: write
+
 jobs:
   analyze:
+    if: github.repository_owner == 'github'
     runs-on: ubuntu-latest-xl
     steps:
       ### Build the queries ###
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@main
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version
@@ -65,7 +70,7 @@ jobs:
             exclude:*/ql/lib/upgrades/
             exclude:java/ql/integration-tests
       - name: Upload sarif to code-scanning
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@main
         with:
           sarif_file: ql-for-ql.sarif
           category: ql-for-ql

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -11,6 +11,10 @@ on:
       - ql/ql/src/ql.dbscheme
   workflow_dispatch:
 
+permissions:
+  contents: read
+  security-events: read
+
 jobs:
   measure:
     env:
@@ -21,11 +25,11 @@ jobs:
           - github/codeql
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@main
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version
@@ -42,7 +46,7 @@ jobs:
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: ${{ matrix.repo }}
           path: ${{ github.workspace }}/repo
@@ -71,7 +75,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: measure
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/download-artifact@v3
         with:
           name: measurements

.github/workflows/ql-for-ql-tests.yml

@@ -17,14 +17,17 @@ on:
 env:
   CARGO_TERM_COLOR: always
 
+permissions:
+  contents: read
+
 jobs:
   qltest:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@main
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version
@@ -61,7 +64,7 @@ jobs:
     needs: [qltest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install GNU tar 
         if: runner.os == 'macOS'
         run: |
@@ -69,7 +72,7 @@ jobs:
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@main
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version

.github/workflows/query-list.yml

@@ -13,6 +13,9 @@ on:
       - '.github/actions/fetch-codeql/action.yml'
       - 'misc/scripts/generate-code-scanning-query-list.py'
 
+permissions:
+  contents: read
+
 jobs:
   build:
 
@@ -20,7 +23,7 @@ jobs:
 
     steps:
     - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         path: codeql 
     - name: Set up Python 3.8

.github/workflows/ruby-build.yml

@@ -32,6 +32,9 @@ defaults:
   run:
     working-directory: ruby
 
+permissions:
+  contents: read
+
 jobs:
   build:
     strategy:
@@ -42,15 +45,17 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install GNU tar
         if: runner.os == 'macOS'
         run: |
           brew install gnu-tar
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
-      - name: Install cargo-cross
-        if: runner.os == 'Linux'
-        run: cargo install cross --version 0.2.5
+      - name: Prepare Windows
+        if: runner.os == 'Windows'
+        shell: powershell
+        run: |
+          git config --global core.longpaths true
       - uses: ./.github/actions/os-version
         id: os_version
       - name: Cache entire extractor
@@ -79,16 +84,8 @@ jobs:
       - name: Run tests
         if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd extractor && cargo test --verbose
-      # On linux, build the extractor via cross in a centos7 container.
-      # This ensures we don't depend on glibc > 2.17.
-      - name: Release build (linux)
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os == 'Linux'
-        run: |
-          cd extractor
-          cross build --release
-          mv target/x86_64-unknown-linux-gnu/release/codeql-extractor-ruby target/release/
-      - name: Release build (windows and macos)
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os != 'Linux'
+      - name: Release build
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd extractor && cargo build --release
       - name: Generate dbscheme
         if: ${{ matrix.os == 'ubuntu-latest' && steps.cache-extractor.outputs.cache-hit != 'true'}}
@@ -111,15 +108,16 @@ jobs:
             ruby/extractor/target/release/codeql-extractor-ruby.exe
           retention-days: 1
   compile-queries:
+    if: github.repository_owner == 'github'
     runs-on: ubuntu-latest-xl
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Fetch CodeQL
         uses: ./.github/actions/fetch-codeql
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with: 
+        with:
           key: ruby-build
       - name: Build Query Pack
         run: |
@@ -145,7 +143,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: [build, compile-queries]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/download-artifact@v3
         with:
           name: ruby.dbscheme
@@ -206,7 +204,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     needs: [package]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Fetch CodeQL
         uses: ./.github/actions/fetch-codeql
 
@@ -231,54 +229,3 @@ jobs:
         shell: bash
         run: |
           codeql database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls
-
-  # This is a copy of the 'test' job that runs in a centos7 container.
-  # This tests that the extractor works correctly on systems with an old glibc.
-  test-centos7:
-    defaults:
-      run:
-        working-directory: ${{ github.workspace }}
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-latest
-    container:
-      image: centos:centos7
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    needs: [package]
-    steps:
-      - name: Install gh cli
-        run: |
-          yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
-          # fetch-codeql requires unzip and jq
-          # jq is available in epel-release (https://docs.fedoraproject.org/en-US/epel/)
-          yum install -y gh unzip epel-release
-          yum install -y jq
-      - uses: actions/checkout@v3
-      - name: Fetch CodeQL
-        uses: ./.github/actions/fetch-codeql
-
-      # Due to a bug in Actions, we can't use runner.temp in the run blocks here.
-      # https://github.com/actions/runner/issues/2185
-
-      - name: Download Ruby bundle
-        uses: actions/download-artifact@v3
-        with:
-          name: codeql-ruby-bundle
-          path: ${{ runner.temp }}
-      - name: Unzip Ruby bundle
-        shell: bash
-        run: unzip -q -d "$RUNNER_TEMP"/ruby-bundle "$RUNNER_TEMP"/codeql-ruby-bundle.zip
-
-      - name: Run QL test
-        shell: bash
-        run: |
-          codeql test run --search-path "$RUNNER_TEMP"/ruby-bundle --additional-packs "$RUNNER_TEMP"/ruby-bundle ruby/ql/test/library-tests/ast/constants/
-      - name: Create database
-        shell: bash
-        run: |
-          codeql database create --search-path "$RUNNER_TEMP"/ruby-bundle --language ruby --source-root ruby/ql/test/library-tests/ast/constants/ ../database
-      - name: Analyze database
-        shell: bash
-        run: |
-          codeql database analyze --search-path "$RUNNER_TEMP"/ruby-bundle --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/ruby-dataset-measure.yml

@@ -17,6 +17,9 @@ on:
       - .github/workflows/ruby-dataset-measure.yml
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   measure:
     env:
@@ -27,14 +30,14 @@ jobs:
         repo: [rails/rails, discourse/discourse, spree/spree, ruby/ruby]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: ./.github/actions/fetch-codeql
 
       - uses: ./ruby/actions/create-extractor-pack
 
       - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: ${{ matrix.repo }}
           path: ${{ github.workspace }}/repo
@@ -59,7 +62,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: measure
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/download-artifact@v3
         with:
           name: measurements

.github/workflows/ruby-qltest.yml

@@ -14,6 +14,7 @@ on:
   pull_request:
     paths:
       - "ruby/**"
+      - "shared/**"
       - .github/workflows/ruby-qltest.yml
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
@@ -28,11 +29,14 @@ defaults:
   run:
     working-directory: ruby
 
+permissions:
+  contents: read
+
 jobs:
   qlupgrade:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/fetch-codeql
       - name: Check DB upgrade scripts
         run: |
@@ -49,11 +53,12 @@ jobs:
            xargs codeql execute upgrades testdb
           diff -q testdb/ruby.dbscheme downgrades/initial/ruby.dbscheme
   qltest:
+    if: github.repository_owner == 'github'
     runs-on: ubuntu-latest-xl
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/fetch-codeql
       - uses: ./ruby/actions/create-extractor-pack
       - name: Cache compilation cache

.github/workflows/swift.yml

@@ -6,6 +6,7 @@ on:
       - "swift/**"
       - "misc/bazel/**"
       - "misc/codegen/**"
+      - "shared/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**
@@ -22,10 +23,12 @@ on:
       - "swift/**"
       - "misc/bazel/**"
       - "misc/codegen/**"
+      - "shared/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**
       - codeql-workspace.yml
+      - .pre-commit-config.yaml
       - "!**/*.md"
       - "!**/*.qhelp"
     branches:
@@ -33,61 +36,77 @@ on:
       - rc/*
       - codeql-cli-*
 
+permissions:
+  contents: read
+
 jobs:
   # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks
   # without waiting for the macOS build
   build-and-test-macos:
+    if: github.repository_owner == 'github'
     runs-on: macos-12-xl
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./swift/actions/build-and-test
   build-and-test-linux:
+    if: github.repository_owner == 'github'
     runs-on: ubuntu-latest-xl
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./swift/actions/build-and-test
   qltests-linux:
+    if: github.repository_owner == 'github'
     needs: build-and-test-linux
     runs-on: ubuntu-latest-xl
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./swift/actions/run-ql-tests
   qltests-macos:
-    if : ${{ github.event_name == 'pull_request' }}
+    if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }}
     needs: build-and-test-macos
     runs-on: macos-12-xl
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./swift/actions/run-ql-tests
   integration-tests-linux:
+    if: github.repository_owner == 'github'
     needs: build-and-test-linux
     runs-on: ubuntu-latest-xl
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./swift/actions/run-integration-tests
   integration-tests-macos:
-    if : ${{ github.event_name == 'pull_request' }}
+    if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }}
     needs: build-and-test-macos
     runs-on: macos-12-xl
     timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./swift/actions/run-integration-tests
+  clang-format:
+    if : ${{ github.event_name == 'pull_request' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
+        name: Check that python code is properly formatted
+        with:
+          extra_args: clang-format --all-files
   codegen:
     if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: bazelbuild/setup-bazelisk@v2
       - uses: actions/setup-python@v4
         with:
           python-version-file: 'swift/.python-version'
-      - uses: pre-commit/action@v3.0.0
+      - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
         name: Check that python code is properly formatted
         with:
           extra_args: autopep8 --all-files
       - uses: ./.github/actions/fetch-codeql
-      - uses: pre-commit/action@v3.0.0
+      - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
         name: Check that QL generated code was checked in
         with:
           extra_args: swift-codegen --all-files
@@ -102,6 +121,6 @@ jobs:
     if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/fetch-codeql
       - uses: ./swift/actions/database-upgrade-scripts

.github/workflows/sync-files.yml

@@ -10,11 +10,14 @@ on:
       - main
       - 'rc/*'
 
+permissions:
+  contents: read
+
 jobs:
   sync:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Check synchronized files
         run: python config/sync-files.py
       - name: Check dbscheme fragments

.github/workflows/tree-sitter-extractor-test.yml

@@ -23,11 +23,14 @@ defaults:
   run:
     working-directory: shared/tree-sitter-extractor
 
+permissions:
+  contents: read
+
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Check formatting
         run: cargo fmt --all -- --check
       - name: Run tests
@@ -35,12 +38,12 @@ jobs:
   fmt:
     runs-on: ubuntu-latest  
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Check formatting
         run: cargo fmt --check
   clippy:
     runs-on: ubuntu-latest  
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Run clippy
         run: cargo clippy -- --no-deps -D warnings -A clippy::new_without_default -A clippy::too_many_arguments

.github/workflows/validate-change-notes.yml

@@ -15,12 +15,15 @@ on:
       - ".github/workflows/validate-change-notes.yml"
       - ".github/actions/fetch-codeql/action.yml"
 
+permissions:
+  contents: read
+
 jobs:
   check-change-note:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql

.gitignore

@@ -39,6 +39,9 @@
 # local bazel options
 /local.bazelrc
 
+# generated cmake directory
+/.bazel-cmake
+
 # CLion project files
 /.clwb
 

.lfsconfig

@@ -0,0 +1,5 @@
+[lfs]
+# codeql is publicly forked by many users, and we don't want any LFS file polluting their working
+# copies. We therefore exclude everything by default.
+# For files required by bazel builds, use rules in `misc/bazel/lfs.bzl` to download them on demand.
+fetchinclude = /nothing

.pre-commit-config.yaml

@@ -5,15 +5,14 @@ repos:
     rev: v3.2.0
     hooks:
       - id: trailing-whitespace
-        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
+        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)|.*\.patch
       - id: end-of-file-fixer
-        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
+        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)|.*\.patch
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v13.0.1
+    rev: v17.0.6
     hooks:
       - id: clang-format
-        files: ^swift/.*\.(h|c|cpp)$
 
   - repo: https://github.com/pre-commit/mirrors-autopep8
     rev: v1.6.0
@@ -23,6 +22,20 @@ repos:
 
   - repo: local
     hooks:
+      - id: buildifier
+        name: Format bazel files
+        files: \.(bazel|bzl)
+        language: system
+        entry: bazel run //misc/bazel:buildifier
+        pass_filenames: false
+
+      - id: go-gen
+        name: Check checked in generated files in go
+        files: ^go/.*
+        language: system
+        entry: bazel run //go:gen
+        pass_filenames: false
+
       - id: codeql-format
         name: Fix QL file formatting
         files: \.qll?$

CODEOWNERS

@@ -1,4 +1,5 @@
 /cpp/ @github/codeql-c-analysis
+/cpp/autobuilder/ @github/codeql-c-extractor
 /csharp/ @github/codeql-csharp
 /go/ @github/codeql-go
 /java/ @github/codeql-java
@@ -8,9 +9,8 @@
 /swift/ @github/codeql-swift
 /misc/codegen/ @github/codeql-swift
 /java/kotlin-extractor/ @github/codeql-kotlin
-
-# ML-powered queries
-/javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
+/java/ql/test-kotlin1/ @github/codeql-kotlin
+/java/ql/test-kotlin2/ @github/codeql-kotlin
 
 # CodeQL tools and associated docs
 /docs/codeql/codeql-cli/ @github/codeql-cli-reviewers
@@ -23,6 +23,7 @@
 
 # Bazel (excluding BUILD.bazel files)
 WORKSPACE.bazel @github/codeql-ci-reviewers
+MODULE.bazel @github/codeql-ci-reviewers
 .bazelversion @github/codeql-ci-reviewers
 .bazelrc @github/codeql-ci-reviewers
 **/*.bzl @github/codeql-ci-reviewers
@@ -33,12 +34,11 @@ WORKSPACE.bazel @github/codeql-ci-reviewers
 
 # Workflows
 /.github/workflows/ @github/codeql-ci-reviewers
-/.github/workflows/atm-* @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/go-* @github/codeql-go
-/.github/workflows/js-ml-tests.yml @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
 /.github/workflows/ruby-* @github/codeql-ruby
 /.github/workflows/swift.yml @github/codeql-swift
 
 # Misc
 /misc/scripts/accept-expected-changes-from-ci.py @RasmusWL
+/misc/scripts/generate-code-scanning-query-list.py @RasmusWL

CONTRIBUTING.md

@@ -4,6 +4,8 @@ We welcome contributions to our CodeQL libraries and queries. Got an idea for a
 
 There is lots of useful documentation to help you write queries, ranging from information about query file structure to tutorials for specific target languages. For more information on the documentation available, see [CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/codeql-queries) on [codeql.github.com](https://codeql.github.com).
 
+Note that the CodeQL for Visual Studio Code documentation has been migrated to https://docs.github.com/en/code-security/codeql-for-vs-code/, but you can still contribute to it via a different repository. For more information, see [Contributing to GitHub Docs documentation](https://docs.github.com/en/contributing)."
+
 ## Change notes
 
 Any nontrivial user-visible change to a query pack or library pack should have a change note. For details on how to add a change note for your change, see [this guide](docs/change-notes.md).
@@ -14,14 +16,16 @@ If you have an idea for a query that you would like to share with other CodeQL u
 
 1. **Directory structure**
 
-    There are six language-specific query directories in this repository:
+    There are eight language-specific query directories in this repository:
 
       * C/C++: `cpp/ql/src`
       * C#: `csharp/ql/src`
-      * Java: `java/ql/src`
+      * Go: `go/ql/src`
+      * Java/Kotlin: `java/ql/src`
       * JavaScript: `javascript/ql/src`
       * Python: `python/ql/src`
       * Ruby: `ruby/ql/src`
+      * Swift: `swift/ql/src`
 
     Each language-specific directory contains further subdirectories that group queries based on their `@tags` or purpose.
     - Experimental queries and libraries are stored in the `experimental` subdirectory within each language-specific directory in the [CodeQL repository](https://github.com/github/codeql). For example, experimental Java queries and libraries are stored in `java/ql/src/experimental` and any corresponding tests in `java/ql/test/experimental`.
@@ -41,7 +45,7 @@ If you have an idea for a query that you would like to share with other CodeQL u
 
 3. **Formatting**
 
-    - The queries and libraries must be autoformatted, for example using the "Format Document" command in [CodeQL for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-codeql-for-visual-studio-code).
+    - The queries and libraries must be autoformatted, for example using the "Format Document" command in [CodeQL for Visual Studio Code](https://docs.github.com/en/code-security/codeql-for-vs-code/).
 
     If you prefer, you can either:
     1. install the [pre-commit framework](https://pre-commit.com/) and install the configured hooks on this repo via `pre-commit install`, or

MODULE.bazel

@@ -0,0 +1,62 @@
+module(
+    name = "codeql",
+    version = "0.0",
+)
+
+# this points to our internal repository when `codeql` is checked out as a submodule thereof
+# when building things from `codeql` independently this is stubbed out in `.bazelrc`
+bazel_dep(name = "semmle_code", version = "0.0")
+local_path_override(
+    module_name = "semmle_code",
+    path = "..",
+)
+
+# see https://registry.bazel.build/ for a list of available packages
+
+bazel_dep(name = "platforms", version = "0.0.9")
+bazel_dep(name = "rules_go", version = "0.47.0")
+bazel_dep(name = "rules_pkg", version = "0.10.1")
+bazel_dep(name = "rules_nodejs", version = "6.0.3")
+bazel_dep(name = "rules_python", version = "0.31.0")
+bazel_dep(name = "bazel_skylib", version = "1.5.0")
+bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl")
+bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
+bazel_dep(name = "fmt", version = "10.0.0")
+bazel_dep(name = "gazelle", version = "0.36.0")
+
+bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
+
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.parse(
+    hub_name = "codegen_deps",
+    python_version = "3.11",
+    requirements_lock = "//misc/codegen:requirements_lock.txt",
+)
+use_repo(pip, "codegen_deps")
+
+swift_deps = use_extension("//swift/third_party:load.bzl", "swift_deps")
+
+# following list can be kept in sync with `bazel mod tidy`
+use_repo(
+    swift_deps,
+    "binlog",
+    "picosha2",
+    "swift_prebuilt_darwin_x86_64",
+    "swift_prebuilt_linux",
+    "swift_toolchain_linux",
+    "swift_toolchain_macos",
+)
+
+node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node")
+node.toolchain(
+    name = "nodejs",
+    node_version = "18.15.0",
+)
+use_repo(node, "nodejs", "nodejs_toolchains")
+
+go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
+go_sdk.download(version = "1.22.2")
+
+register_toolchains(
+    "@nodejs_toolchains//:all",
+)

README.md

@@ -4,7 +4,7 @@ This open source repository contains the standard CodeQL libraries and queries t
 
 ## How do I learn CodeQL and run queries?
 
-There is [extensive documentation](https://codeql.github.com/docs/) on getting started with writing CodeQL using the [CodeQL extension for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/) and the [CodeQL CLI](https://codeql.github.com/docs/codeql-cli/).
+There is extensive documentation about the [CodeQL language](https://codeql.github.com/docs/), writing CodeQL using the [CodeQL extension for Visual Studio Code](https://docs.github.com/en/code-security/codeql-for-vs-code/) and using the [CodeQL CLI](https://docs.github.com/en/code-security/codeql-cli).
 
 ## Contributing
 

WORKSPACE.bazel

@@ -1,12 +1,2 @@
-# Please notice that any bazel targets and definitions in this repository are currently experimental
-# and for internal use only.
-
-workspace(name = "codeql")
-
-load("//misc/bazel:workspace.bzl", "codeql_workspace")
-
-codeql_workspace()
-
-load("//misc/bazel:workspace_deps.bzl", "codeql_workspace_deps")
-
-codeql_workspace_deps()
+# please use MODULE.bazel to add dependencies
+# this empty file is required by internal repositories, don't remove it

codeql-workspace.yml

@@ -1,23 +1,16 @@
 provide:
   - "*/ql/src/qlpack.yml"
   - "*/ql/lib/qlpack.yml"
-  - "*/ql/test/qlpack.yml"
+  - "*/ql/test*/qlpack.yml"
   - "*/ql/examples/qlpack.yml"
   - "*/ql/consistency-queries/qlpack.yml"
-  - "shared/*/qlpack.yml"
+  - "*/ql/automodel/src/qlpack.yml"
+  - "*/ql/automodel/test/qlpack.yml"
+  - "python/extractor/qlpack.yml"
+  - "shared/**/qlpack.yml"
   - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
   - "go/ql/config/legacy-support/qlpack.yml"
   - "go/build/codeql-extractor-go/codeql-extractor.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml"
-  # This pack is explicitly excluded from the workspace since most users
-  # will want to use a version of this pack from the package cache. Internal
-  # users can uncomment the following line and place a custom ML model
-  # in the corresponding pack to test a custom ML model within their local
-  # checkout.
-  # - "javascript/ql/experimental/adaptivethreatmodeling/model/qlpack.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/test/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/lib/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/src/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/test/qlpack.yml"
@@ -25,8 +18,8 @@ provide:
   - "misc/suite-helpers/qlpack.yml"
   - "ruby/extractor-pack/codeql-extractor.yml"
   - "swift/extractor-pack/codeql-extractor.yml"
-  - "swift/integration-tests/qlpack.yml"
   - "ql/extractor-pack/codeql-extractor.yml"
+  - ".github/codeql/extensions/**/codeql-pack.yml"
 
 versionPolicies:
   default:

config/identical-files.json

@@ -1,24 +1,4 @@
 {
-  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift": [
-    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlow.qll",
-    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlow.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlow.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlow.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlow.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlow.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlow.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlow.qll"
-  ],
-  "DataFlowImpl Java/C++/C#/Go/Python/Ruby/Swift": [
-    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll",
-    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"
-  ],
   "DataFlow Java/C++/C#/Go/Python/Ruby/Swift Legacy Configuration": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl1.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl2.qll",
@@ -42,37 +22,14 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll",
     "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl1.qll",
     "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl2.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImplForStringsNewReplacer.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl1.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl1.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForHttpClientLibraries.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForPathname.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl1.qll"
   ],
-  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift Common": [
-    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll",
-    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImplCommon.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll"
-  ],
-  "TaintTracking Java/C++/C#/Go/Python/Ruby/Swift": [
-    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTracking.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTracking.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking1/TaintTracking.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/tainttracking1/TaintTracking.qll",
-    "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTracking.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking1/TaintTracking.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTracking.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/tainttracking1/TaintTracking.qll"
-  ],
   "TaintTracking Legacy Configuration Java/C++/C#/Go/Python/Ruby/Swift": [
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
@@ -96,23 +53,6 @@
     "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
   ],
-  "DataFlow Java/C++/C#/Python/Ruby/Swift Consistency checks": [
-    "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll",
-    "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplConsistency.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplConsistency.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplConsistency.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplConsistency.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplConsistency.qll"
-  ],
-  "DataFlow Java/C#/Go/Ruby/Python/Swift Flow Summaries": [
-    "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll",
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/FlowSummaryImpl.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/FlowSummaryImpl.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/FlowSummaryImpl.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/FlowSummaryImpl.qll"
-  ],
   "SsaReadPosition Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll"
@@ -148,123 +88,46 @@
   "IR Instruction": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/Instruction.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/Instruction.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/Instruction.qll",
-    "csharp/ql/src/experimental/ir/implementation/raw/Instruction.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/Instruction.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/Instruction.qll"
   ],
   "IR IRBlock": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/IRBlock.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/IRBlock.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/IRBlock.qll",
-    "csharp/ql/src/experimental/ir/implementation/raw/IRBlock.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/IRBlock.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/IRBlock.qll"
   ],
   "IR IRVariable": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/IRVariable.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/IRVariable.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/IRVariable.qll",
-    "csharp/ql/src/experimental/ir/implementation/raw/IRVariable.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/IRVariable.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/IRVariable.qll"
   ],
   "IR IRFunction": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/IRFunction.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/IRFunction.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/IRFunction.qll",
-    "csharp/ql/src/experimental/ir/implementation/raw/IRFunction.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/IRFunction.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/IRFunction.qll"
   ],
   "IR Operand": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/Operand.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/Operand.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/Operand.qll",
-    "csharp/ql/src/experimental/ir/implementation/raw/Operand.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/Operand.qll"
-  ],
-  "IR IRType": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/IRType.qll",
-    "csharp/ql/src/experimental/ir/implementation/IRType.qll"
-  ],
-  "IR IRConfiguration": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/IRConfiguration.qll",
-    "csharp/ql/src/experimental/ir/implementation/IRConfiguration.qll"
-  ],
-  "IR UseSoundEscapeAnalysis": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/UseSoundEscapeAnalysis.qll",
-    "csharp/ql/src/experimental/ir/implementation/UseSoundEscapeAnalysis.qll"
-  ],
-  "IR IRFunctionBase": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/internal/IRFunctionBase.qll",
-    "csharp/ql/src/experimental/ir/implementation/internal/IRFunctionBase.qll"
-  ],
-  "IR Operand Tag": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/internal/OperandTag.qll",
-    "csharp/ql/src/experimental/ir/implementation/internal/OperandTag.qll"
-  ],
-  "IR TInstruction": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/internal/TInstruction.qll",
-    "csharp/ql/src/experimental/ir/implementation/internal/TInstruction.qll"
-  ],
-  "IR TIRVariable": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/internal/TIRVariable.qll",
-    "csharp/ql/src/experimental/ir/implementation/internal/TIRVariable.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/Operand.qll"
   ],
   "IR IR": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/IR.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/IR.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/IR.qll",
-    "csharp/ql/src/experimental/ir/implementation/raw/IR.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/IR.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/IR.qll"
   ],
   "IR IRConsistency": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/IRConsistency.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/IRConsistency.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/IRConsistency.qll",
-    "csharp/ql/src/experimental/ir/implementation/raw/IRConsistency.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/IRConsistency.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/IRConsistency.qll"
   ],
   "IR PrintIR": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/PrintIR.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/PrintIR.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/PrintIR.qll",
-    "csharp/ql/src/experimental/ir/implementation/raw/PrintIR.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/PrintIR.qll"
-  ],
-  "IR IntegerConstant": [
-    "cpp/ql/lib/semmle/code/cpp/ir/internal/IntegerConstant.qll",
-    "csharp/ql/src/experimental/ir/internal/IntegerConstant.qll"
-  ],
-  "IR IntegerInteval": [
-    "cpp/ql/lib/semmle/code/cpp/ir/internal/IntegerInterval.qll",
-    "csharp/ql/src/experimental/ir/internal/IntegerInterval.qll"
-  ],
-  "IR IntegerPartial": [
-    "cpp/ql/lib/semmle/code/cpp/ir/internal/IntegerPartial.qll",
-    "csharp/ql/src/experimental/ir/internal/IntegerPartial.qll"
-  ],
-  "IR Overlap": [
-    "cpp/ql/lib/semmle/code/cpp/ir/internal/Overlap.qll",
-    "csharp/ql/src/experimental/ir/internal/Overlap.qll"
-  ],
-  "IR EdgeKind": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/EdgeKind.qll",
-    "csharp/ql/src/experimental/ir/implementation/EdgeKind.qll"
-  ],
-  "IR MemoryAccessKind": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/MemoryAccessKind.qll",
-    "csharp/ql/src/experimental/ir/implementation/MemoryAccessKind.qll"
-  ],
-  "IR TempVariableTag": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/TempVariableTag.qll",
-    "csharp/ql/src/experimental/ir/implementation/TempVariableTag.qll"
-  ],
-  "IR Opcode": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/Opcode.qll",
-    "csharp/ql/src/experimental/ir/implementation/Opcode.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/PrintIR.qll"
   ],
   "IR SSAConsistency": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/SSAConsistency.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/SSAConsistency.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/internal/SSAConsistency.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/SSAConsistency.qll"
   ],
   "C++ IR InstructionImports": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/InstructionImports.qll",
@@ -312,8 +175,7 @@
   ],
   "SSA AliasAnalysis": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/AliasAnalysis.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/AliasAnalysis.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/internal/AliasAnalysis.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/AliasAnalysis.qll"
   ],
   "SSA PrintAliasAnalysis": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/PrintAliasAnalysis.qll",
@@ -328,44 +190,28 @@
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/gvn/internal/ValueNumberingImports.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/gvn/internal/ValueNumberingImports.qll"
   ],
-  "IR SSA SimpleSSA": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/SimpleSSA.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/internal/SimpleSSA.qll"
-  ],
-  "IR AliasConfiguration (unaliased_ssa)": [
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/AliasConfiguration.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/internal/AliasConfiguration.qll"
-  ],
   "IR SSA SSAConstruction": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/SSAConstruction.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/SSAConstruction.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/internal/SSAConstruction.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/SSAConstruction.qll"
   ],
   "IR SSA PrintSSA": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/PrintSSA.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/PrintSSA.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/internal/PrintSSA.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/PrintSSA.qll"
   ],
   "IR ValueNumberInternal": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/gvn/internal/ValueNumberingInternal.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/gvn/internal/ValueNumberingInternal.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/gvn/internal/ValueNumberingInternal.qll",
-    "csharp/ql/src/experimental/ir/implementation/raw/gvn/internal/ValueNumberingInternal.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/gvn/internal/ValueNumberingInternal.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/gvn/internal/ValueNumberingInternal.qll"
   ],
   "C++ IR ValueNumber": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/gvn/ValueNumbering.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/gvn/ValueNumbering.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/gvn/ValueNumbering.qll",
-    "csharp/ql/src/experimental/ir/implementation/raw/gvn/ValueNumbering.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/gvn/ValueNumbering.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/gvn/ValueNumbering.qll"
   ],
   "C++ IR PrintValueNumbering": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/gvn/PrintValueNumbering.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/gvn/PrintValueNumbering.qll",
-    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/gvn/PrintValueNumbering.qll",
-    "csharp/ql/src/experimental/ir/implementation/raw/gvn/PrintValueNumbering.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/gvn/PrintValueNumbering.qll"
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/gvn/PrintValueNumbering.qll"
   ],
   "C++ IR ConstantAnalysis": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/constant/ConstantAnalysis.qll",
@@ -393,38 +239,6 @@
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/reachability/PrintDominance.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/reachability/PrintDominance.qll"
   ],
-  "C# IR InstructionImports": [
-    "csharp/ql/src/experimental/ir/implementation/raw/internal/InstructionImports.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/internal/InstructionImports.qll"
-  ],
-  "C# IR IRImports": [
-    "csharp/ql/src/experimental/ir/implementation/raw/internal/IRImports.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/internal/IRImports.qll"
-  ],
-  "C# IR IRBlockImports": [
-    "csharp/ql/src/experimental/ir/implementation/raw/internal/IRBlockImports.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/internal/IRBlockImports.qll"
-  ],
-  "C# IR IRFunctionImports": [
-    "csharp/ql/src/experimental/ir/implementation/raw/internal/IRFunctionImports.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/internal/IRFunctionImports.qll"
-  ],
-  "C# IR IRVariableImports": [
-    "csharp/ql/src/experimental/ir/implementation/raw/internal/IRVariableImports.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/internal/IRVariableImports.qll"
-  ],
-  "C# IR OperandImports": [
-    "csharp/ql/src/experimental/ir/implementation/raw/internal/OperandImports.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/internal/OperandImports.qll"
-  ],
-  "C# IR PrintIRImports": [
-    "csharp/ql/src/experimental/ir/implementation/raw/internal/PrintIRImports.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/internal/PrintIRImports.qll"
-  ],
-  "C# IR ValueNumberingImports": [
-    "csharp/ql/src/experimental/ir/implementation/raw/gvn/internal/ValueNumberingImports.qll",
-    "csharp/ql/src/experimental/ir/implementation/unaliased_ssa/gvn/internal/ValueNumberingImports.qll"
-  ],
   "C# ControlFlowReachability": [
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/ControlFlowReachability.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/ControlFlowReachability.qll"
@@ -437,13 +251,6 @@
     "cpp/ql/src/Security/CWE/CWE-020/SafeExternalAPIFunction.qll",
     "cpp/ql/src/Security/CWE/CWE-020/ir/SafeExternalAPIFunction.qll"
   ],
-  "XML": [
-    "cpp/ql/lib/semmle/code/cpp/XML.qll",
-    "csharp/ql/lib/semmle/code/csharp/XML.qll",
-    "java/ql/lib/semmle/code/xml/XML.qll",
-    "javascript/ql/lib/semmle/javascript/XML.qll",
-    "python/ql/lib/semmle/python/xml/XML.qll"
-  ],
   "DuplicationProblems.inc.qhelp": [
     "cpp/ql/src/Metrics/Files/DuplicationProblems.inc.qhelp",
     "javascript/ql/src/Metrics/DuplicationProblems.inc.qhelp",
@@ -491,13 +298,6 @@
     "java/ql/src/experimental/Security/CWE/CWE-400/LocalThreadResourceAbuse.qhelp",
     "java/ql/src/experimental/Security/CWE/CWE-400/ThreadResourceAbuse.qhelp"
   ],
-  "IDE Contextual Queries": [
-    "cpp/ql/lib/IDEContextual.qll",
-    "csharp/ql/lib/IDEContextual.qll",
-    "java/ql/lib/IDEContextual.qll",
-    "javascript/ql/lib/IDEContextual.qll",
-    "python/ql/lib/analysis/IDEContextual.qll"
-  ],
   "CryptoAlgorithms Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/CryptoAlgorithms.qll",
     "python/ql/lib/semmle/python/concepts/CryptoAlgorithms.qll",
@@ -514,24 +314,6 @@
     "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll",
     "swift/ql/lib/codeql/swift/security/internal/SensitiveDataHeuristics.qll"
   ],
-  "CFG": [
-    "csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll",
-    "ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll",
-    "swift/ql/lib/codeql/swift/controlflow/internal/ControlFlowGraphImplShared.qll"
-  ],
-  "TypeTracker": [
-    "python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll",
-    "ruby/ql/lib/codeql/ruby/typetracking/TypeTracker.qll"
-  ],
-  "AccessPathSyntax": [
-    "csharp/ql/lib/semmle/code/csharp/dataflow/internal/AccessPathSyntax.qll",
-    "go/ql/lib/semmle/go/dataflow/internal/AccessPathSyntax.qll",
-    "java/ql/lib/semmle/code/java/dataflow/internal/AccessPathSyntax.qll",
-    "javascript/ql/lib/semmle/javascript/frameworks/data/internal/AccessPathSyntax.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/AccessPathSyntax.qll",
-    "swift/ql/lib/codeql/swift/dataflow/internal/AccessPathSyntax.qll"
-  ],
   "IncompleteUrlSubstringSanitization": [
     "javascript/ql/src/Security/CWE-020/IncompleteUrlSubstringSanitization.qll",
     "ruby/ql/src/queries/security/cwe-020/IncompleteUrlSubstringSanitization.qll"
@@ -551,26 +333,6 @@
     "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModelsExtensions.qll",
     "python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsExtensions.qll"
   ],
-  "TaintedFormatStringQuery Ruby/JS": [
-    "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringQuery.qll",
-    "ruby/ql/lib/codeql/ruby/security/TaintedFormatStringQuery.qll"
-  ],
-  "TaintedFormatStringCustomizations Ruby/JS": [
-    "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringCustomizations.qll",
-    "ruby/ql/lib/codeql/ruby/security/TaintedFormatStringCustomizations.qll"
-  ],
-  "HttpToFileAccessQuery JS/Ruby": [
-    "javascript/ql/lib/semmle/javascript/security/dataflow/HttpToFileAccessQuery.qll",
-    "ruby/ql/lib/codeql/ruby/security/HttpToFileAccessQuery.qll"
-  ],
-  "HttpToFileAccessCustomizations JS/Ruby": [
-    "javascript/ql/lib/semmle/javascript/security/dataflow/HttpToFileAccessCustomizations.qll",
-    "ruby/ql/lib/codeql/ruby/security/HttpToFileAccessCustomizations.qll"
-  ],
-  "Typo database": [
-    "javascript/ql/src/Expressions/TypoDatabase.qll",
-    "ql/ql/src/codeql_ql/style/TypoDatabase.qll"
-  ],
   "Swift declarations test file": [
     "swift/ql/test/extractor-tests/declarations/declarations.swift",
     "swift/ql/test/library-tests/ast/declarations.swift"
@@ -598,5 +360,9 @@
   "EncryptionKeySizes Python/Java": [
     "python/ql/lib/semmle/python/security/internal/EncryptionKeySizes.qll",
     "java/ql/lib/semmle/code/java/security/internal/EncryptionKeySizes.qll"
+  ],
+  "Python model summaries test extension": [
+    "python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.ext.yml",
+    "python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ext.yml"
   ]
 }

cpp/BUILD.bazel

@@ -1,12 +1,17 @@
-package(default_visibility = ["//visibility:public"])
+load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup")
 
-load("@rules_pkg//:mappings.bzl", "pkg_filegroup")
+package(default_visibility = ["//visibility:public"])
 
 alias(
     name = "dbscheme",
     actual = "//cpp/ql/lib:dbscheme",
 )
 
+alias(
+    name = "dbscheme-stats",
+    actual = "//cpp/ql/lib:dbscheme-stats",
+)
+
 pkg_filegroup(
     name = "db-files",
     srcs = [

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -145,9 +145,9 @@ namespace Semmle.Autobuild.Cpp.Tests
 
         bool IBuildActions.IsMacOs() => IsMacOs;
 
-        public bool IsArm { get; set; }
+        public bool IsRunningOnAppleSilicon { get; set; }
 
-        bool IBuildActions.IsArm() => IsArm;
+        bool IBuildActions.IsRunningOnAppleSilicon() => IsRunningOnAppleSilicon;
 
         string IBuildActions.PathCombine(params string[] parts)
         {
@@ -203,6 +203,8 @@ namespace Semmle.Autobuild.Cpp.Tests
         public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
 
         public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
+
+        public void Dispose() { }
     }
 
     /// <summary>
@@ -250,12 +252,7 @@ namespace Semmle.Autobuild.Cpp.Tests
             EndCallbackIn.Add(s);
         }
 
-        CppAutobuilder CreateAutoBuilder(bool isWindows,
-            string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
-            string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
-            string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
-            string? nugetRestore = null, string? allSolutions = null,
-            string cwd = @"C:\Project")
+        CppAutobuilder CreateAutoBuilder(bool isWindows, string? dotnetVersion = null, string cwd = @"C:\Project")
         {
             string codeqlUpperLanguage = Language.Cpp.UpperCaseName;
             Actions.GetEnvironmentVariable[$"CODEQL_AUTOBUILDER_{codeqlUpperLanguage}_NO_INDEXING"] = "false";
@@ -265,22 +262,7 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
-            Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
-            Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
-            Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
-            Actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore;
+            Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION"] = dotnetVersion;
             Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null;
             Actions.GetCurrentDirectory = cwd;
             Actions.IsWindows = isWindows;
@@ -326,8 +308,8 @@ namespace Semmle.Autobuild.Cpp.Tests
         public void TestCppAutobuilderSuccess()
         {
             Actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test.sln -DisableParallelProcessing"] = 1;
-            Actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test.sln -DisableParallelProcessing"] = 0;
-            Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && msbuild C:\Project\test.sln /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"""] = 0;
+            Actions.RunProcess[@"cmd.exe /C scratch\.nuget\nuget.exe restore C:\Project\test.sln -DisableParallelProcessing"] = 0;
+            Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program^ Files^ ^(x86^)\Microsoft^ Visual^ Studio^ 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && msbuild C:\Project\test.sln /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"""] = 0;
             Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0;
@@ -337,10 +319,11 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = true;
             Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
             Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true;
+            Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CPP_SCRATCH_DIR"] = "scratch";
             Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.slx";
             Actions.EnumerateDirectories[@"C:\Project"] = "";
-            Actions.CreateDirectories.Add(@"C:\Project\.nuget");
-            Actions.DownloadFiles.Add(("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", @"C:\Project\.nuget\nuget.exe"));
+            Actions.CreateDirectories.Add(@"scratch\.nuget");
+            Actions.DownloadFiles.Add(("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", @"scratch\.nuget\nuget.exe"));
 
             var autobuilder = CreateAutoBuilder(true);
             var solution = new TestSolution(@"C:\Project\test.sln");

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
     <Nullable>enable</Nullable>
@@ -11,12 +11,12 @@
   <ItemGroup>
     <PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
     <PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
-    <PackageReference Include="xunit" Version="2.4.2" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+    <PackageReference Include="xunit" Version="2.6.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.4">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
   </ItemGroup>
 
   <ItemGroup>

cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs

@@ -26,9 +26,6 @@ namespace Semmle.Autobuild.Cpp
 
         public override BuildScript GetBuildScript()
         {
-            if (Options.BuildCommand != null)
-                return new BuildCommandRule((_, f) => f(null)).Analyse(this, false);
-
             return
                 // First try MSBuild
                 new MsBuildRule().Analyse(this, true) |

cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs

@@ -1,5 +1,7 @@
 using System;
+
 using Semmle.Autobuild.Shared;
+using Semmle.Util;
 
 namespace Semmle.Autobuild.Cpp
 {
@@ -15,7 +17,7 @@ namespace Semmle.Autobuild.Cpp
                 try
                 {
                     Console.WriteLine("CodeQL C++ autobuilder");
-                    var builder = new CppAutobuilder(actions, options);
+                    using var builder = new CppAutobuilder(actions, options);
                     return builder.AttemptBuild();
                 }
                 catch (InvalidEnvironmentException ex)

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <AssemblyName>Semmle.Autobuild.Cpp</AssemblyName>
     <RootNamespace>Semmle.Autobuild.Cpp</RootNamespace>
     <ApplicationIcon />
@@ -17,7 +17,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Build" Version="17.3.2" />
+    <PackageReference Include="Microsoft.Build" Version="17.8.3" />
   </ItemGroup>
 
   <ItemGroup>

cpp/downgrades/0a9eb01d3650642e013eb86be45d952289537f91/upgrade.properties

@@ -0,0 +1,3 @@
+description: Expose whether a function was prototyped or not
+compatibility: backwards
+function_prototyped.rel: delete

cpp/downgrades/298438feb146335af824002589cd6d4e96e5dbf9/exprparents.ql

@@ -0,0 +1,19 @@
+class Element extends @element {
+  string toString() { none() }
+}
+
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Stmt extends @stmt {
+  string toString() { none() }
+}
+
+predicate isStmtWithInitializer(Stmt stmt) { exists(int kind | stmts(stmt, kind, _) | kind = 29) }
+
+from Expr child, int index, int index_new, Element parent
+where
+  exprparents(child, index, parent) and
+  if isStmtWithInitializer(parent) then index_new = index - 1 else index_new = index
+select child, index_new, parent

cpp/downgrades/298438feb146335af824002589cd6d4e96e5dbf9/for_initialization.ql

@@ -0,0 +1,9 @@
+class Stmt extends @stmt {
+  string toString() { none() }
+}
+
+from Stmt f, Stmt i
+where
+  for_initialization(f, i) and
+  f instanceof @stmt_for
+select f, i

cpp/downgrades/298438feb146335af824002589cd6d4e96e5dbf9/stmtparents.ql

@@ -0,0 +1,20 @@
+class Element extends @element {
+  string toString() { none() }
+}
+
+class Stmt extends @stmt {
+  string toString() { none() }
+}
+
+predicate isStmtWithInitializer(Stmt stmt) { exists(int kind | stmts(stmt, kind, _) | kind = 29) }
+
+from Stmt child, int index, int index_new, Element parent
+where
+  stmtparents(child, index, parent) and
+  (
+    not isStmtWithInitializer(parent)
+    or
+    index > 0
+  ) and
+  if isStmtWithInitializer(parent) then index_new = index - 1 else index_new = index
+select child, index_new, parent

cpp/downgrades/298438feb146335af824002589cd6d4e96e5dbf9/upgrade.properties

@@ -0,0 +1,5 @@
+description: Support C++20 range-based for initializers
+compatibility: partial
+exprparents.rel: run exprparents.qlo
+stmtparents.rel: run stmtparents.qlo
+for_initialization.rel: run for_initialization.qlo

cpp/downgrades/4f9fabab5124d49108782c081579f45a70571d74/mangled_name.ql

@@ -0,0 +1,11 @@
+class Declaration extends @declaration {
+  string toString() { none() }
+}
+
+class MangledName extends @mangledname {
+  string toString() { none() }
+}
+
+from Declaration d, MangledName n
+where mangled_name(d, n, _)
+select d, n

cpp/downgrades/4f9fabab5124d49108782c081579f45a70571d74/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add completness information to mangled name table
+compatibility: full
+mangled_name.rel: run mangled_name.qlo

cpp/downgrades/5b388693c66db1e7dc2e76a90aa67a2b6eb74f0f/builtintypes.ql

@@ -0,0 +1,19 @@
+class BuiltinType extends @builtintype {
+  string toString() { none() }
+}
+
+from BuiltinType type, string name, int kind, int kind_new, int size, int sign, int alignment
+where
+  builtintypes(type, name, kind, size, sign, alignment) and
+  if
+    type instanceof @fp16 or
+    type instanceof @std_bfloat16 or
+    type instanceof @std_float16 or
+    type instanceof @complex_std_float32 or
+    type instanceof @complex_float32x or
+    type instanceof @complex_std_float64 or
+    type instanceof @complex_float64x or
+    type instanceof @complex_std_float128
+  then kind_new = 2
+  else kind_new = kind
+select type, name, kind_new, size, sign, alignment

cpp/downgrades/5b388693c66db1e7dc2e76a90aa67a2b6eb74f0f/upgrade.properties

@@ -0,0 +1,3 @@
+description: Introduce new floating-point types from C23 and C++23
+compatibility: backwards
+builtintypes.rel: run builtintypes.qlo

cpp/downgrades/7f34caf73ca98314885030cc5a22b6e328fe687c/functions.ql

@@ -0,0 +1,9 @@
+class Function extends @function {
+  string toString() { none() }
+}
+
+from Function fun, string name, int kind, int kind_new
+where
+  functions(fun, name, kind) and
+  if kind = 7 or kind = 8 then kind_new = 0 else kind_new = kind
+select fun, name, kind_new

cpp/downgrades/7f34caf73ca98314885030cc5a22b6e328fe687c/upgrade.properties

@@ -0,0 +1,3 @@
+description: Support more function types
+compatibility: full
+functions.rel: run functions.qlo

cpp/downgrades/8cba93a44180e0d50a80a660950800d822b981fc/upgrade.properties

@@ -0,0 +1,2 @@
+description: Removed @assignpaddexpr and @assignpsubexpr from @assign_bitwise_expr
+compatibility: full

cpp/downgrades/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix")
 
 pkg_files(
     name = "downgrades",

cpp/downgrades/aa7ff0ab32cd4674f6ab731d32fea64116997b05/exprs.ql

@@ -0,0 +1,13 @@
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Location extends @location_expr {
+  string toString() { none() }
+}
+
+from Expr expr, int kind, int kind_new, Location loc
+where
+  exprs(expr, kind, loc) and
+  if kind = 363 then kind_new = 1 else kind_new = kind
+select expr, kind_new, loc

cpp/downgrades/aa7ff0ab32cd4674f6ab731d32fea64116997b05/upgrade.properties

@@ -0,0 +1,4 @@
+description: Introduce re-use expressions
+compatibility: partial
+expr_reuse.rel: delete
+exprs.rel: run exprs.qlo

cpp/downgrades/abfce5c170f93e281948f7689ece373464fdaf87/expr_reuse.ql

@@ -0,0 +1,7 @@
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+from Expr reuse, Expr original
+where expr_reuse(reuse, original, _)
+select reuse, original

cpp/downgrades/abfce5c170f93e281948f7689ece373464fdaf87/expr_types.ql

@@ -0,0 +1,22 @@
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Type extends @type {
+  string toString() { none() }
+}
+
+predicate existingType(Expr expr, Type type, int value_category) {
+  expr_types(expr, type, value_category)
+}
+
+predicate reuseType(Expr reuse, Type type, int value_category) {
+  exists(Expr original |
+    expr_reuse(reuse, original, value_category) and
+    expr_types(original, type, _)
+  )
+}
+
+from Expr expr, Type type, int value_category
+where existingType(expr, type, value_category) or reuseType(expr, type, value_category)
+select expr, type, value_category

cpp/downgrades/abfce5c170f93e281948f7689ece373464fdaf87/upgrade.properties

@@ -0,0 +1,4 @@
+description: Add value category to expr_reuse table
+compatibility: full
+expr_reuse.rel: run expr_reuse.qlo
+expr_types.rel: run expr_types.qlo

cpp/downgrades/cf72c8898d19eb1b3374432cf79d8276cb07ad43/upgrade.properties

@@ -1,5 +1,6 @@
 description: Support C++17 if and switch initializers
 compatibility: partial
+constexpr_if_initialization.rel: delete
 if_initialization.rel: delete
 switch_initialization.rel: delete
 exprparents.rel: run exprparents.qlo

cpp/downgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/upgrade.properties

@@ -0,0 +1,2 @@
+description: Remove _Float128 type
+compatibility: full

cpp/downgrades/d8149ca90e695fe26f9a0c5a7fa0edd6d4ea3f5d/attribute_args.ql

@@ -0,0 +1,17 @@
+class AttributeArg extends @attribute_arg {
+  string toString() { none() }
+}
+
+class Attribute extends @attribute {
+  string toString() { none() }
+}
+
+class Location extends @location_default {
+  string toString() { none() }
+}
+
+from AttributeArg arg, int kind, int kind_new, Attribute attr, int index, Location location
+where
+  attribute_args(arg, kind, attr, index, location) and
+  if arg instanceof @attribute_arg_expr then kind_new = 0 else kind_new = kind
+select arg, kind_new, attr, index, location

cpp/downgrades/d8149ca90e695fe26f9a0c5a7fa0edd6d4ea3f5d/upgrade.properties

@@ -0,0 +1,4 @@
+description: Support expression attribute arguments
+compatibility: partial
+attribute_arg_expr.rel: delete
+attribute_args.rel: run attribute_args.qlo