Merge branch 'js/expose-internal-module-naming' into js/vea-hacking

This reverts commit 5ed2e033f1.

.bazelrc

@@ -1,4 +1,5 @@
 common --enable_platform_specific_config
+common --enable_bzlmod
 # because we use --override_module with `%workspace%`, the lock file is not stable
 common --lockfile_mode=off
 
@@ -9,20 +10,8 @@ common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
 
 build --repo_env=CC=clang --repo_env=CXX=clang++
 
-# we use transitions that break builds of `...`, so for `test` to work with that we need the following
-test --build_tests_only
-
-# this requires developer mode, but is required to have pack installer functioning
-startup --windows_enable_symlinks
-common --enable_runfiles
-
-# with the above, we can avoid building python zips which is the default on windows as that's expensive
-build --nobuild_python_zip
-
-common --registry=file:///%workspace%/misc/bazel/registry
-common --registry=https://bcr.bazel.build
-
-common --@rules_dotnet//dotnet/settings:strict_deps=false
-common --experimental_isolated_extension_usages
+build:linux --cxxopt=-std=c++20
+build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
+build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
 
 try-import %workspace%/local.bazelrc

.bazelrc.internal

@@ -1,11 +0,0 @@
-# this file should contain bazel settings required to build things from `semmle-code`
-
-common --registry=file:///%workspace%/ql/misc/bazel/registry
-common --registry=https://bcr.bazel.build
-
-# See bazelbuild/rules_dotnet#413: strict_deps in C# also appliy to 3rd-party deps, and when we pull
-# in (for example) the xunit package, there's no code in this at all, it just depends transitively on
-# its implementation packages without providing any code itself.
-# We either can depend on internal implementation details, or turn of strict deps.
-common --@rules_dotnet//dotnet/settings:strict_deps=false
-common --experimental_isolated_extension_usages

.bazelversion

@@ -1 +1 @@
-8.0.0rc1
+7.1.0

.devcontainer/swift/root.sh

@@ -3,16 +3,6 @@ set -xe
 BAZELISK_VERSION=v1.12.0
 BAZELISK_DOWNLOAD_SHA=6b0bcb2ea15bca16fffabe6fda75803440375354c085480fe361d2cbf32501db
 
-# install git lfs apt source
-curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
-
-# install gh apt source
-(type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) \
-&& sudo mkdir -p -m 755 /etc/apt/keyrings \
-&& wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
-&& sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
-&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
-
 apt-get update
 export DEBIAN_FRONTEND=noninteractive
 apt-get -y install --no-install-recommends \
@@ -20,9 +10,7 @@ apt-get -y install --no-install-recommends \
     uuid-dev \
     python3-distutils \
     python3-pip \
-    bash-completion \
-    git-lfs \
-    gh
+    bash-completion
 
 # Install Bazel
 curl -fSsL -o /usr/local/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-amd64

.devcontainer/swift/user.sh

@@ -1,7 +1,5 @@
 set -xe
 
-git lfs install
-
 # add the workspace to the codeql search path
 mkdir -p /home/vscode/.config/codeql
 echo "--search-path /workspaces/codeql" > /home/vscode/.config/codeql/config

.gitattributes

@@ -50,40 +50,26 @@
 *.dll -text
 *.pdb -text
 
-/java/ql/test/stubs/**/*.java linguist-generated=true
-/java/ql/test/experimental/stubs/**/*.java linguist-generated=true
-/java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text
+java/ql/test/stubs/**/*.java linguist-generated=true
+java/ql/test/experimental/stubs/**/*.java linguist-generated=true
 
 # Force git not to modify line endings for go or html files under the go/ql directory
-/go/ql/**/*.go -text
-/go/ql/**/*.html -text
+go/ql/**/*.go -text
+go/ql/**/*.html -text
 # Force git not to modify line endings for go dbschemes
-/go/*.dbscheme -text
+go/*.dbscheme -text
 # Preserve unusual line ending from codeql-go merge
-/go/extractor/opencsv/CSVReader.java -text
+go/extractor/opencsv/CSVReader.java -text
 
 # For some languages, upgrade script testing references really old dbscheme
 # files from legacy upgrades that have CRLF line endings. Since upgrade
 # resolution relies on object hashes, we must suppress line ending conversion
 # for those testing dbscheme files.
-/*/ql/lib/upgrades/initial/*.dbscheme -text
+*/ql/lib/upgrades/initial/*.dbscheme -text
 
 # Auto-generated modeling for Python
-/python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
+python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
 
 # auto-generated bazel lock file
-/ruby/extractor/cargo-bazel-lock.json linguist-generated=true
-/ruby/extractor/cargo-bazel-lock.json -merge
-
-# auto-generated files for the C# build
-/csharp/paket.lock linguist-generated=true
-# needs eol=crlf, as `paket` touches this file and saves it as crlf
-/csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf
-/csharp/paket.main.bzl linguist-generated=true
-/csharp/paket.main_extension.bzl linguist-generated=true
-
-# ripunzip tool
-/misc/ripunzip/ripunzip-* filter=lfs diff=lfs merge=lfs -text
-
-# swift prebuilt resources
-/swift/third_party/resource-dir/*.zip filter=lfs diff=lfs merge=lfs -text
+ruby/extractor/cargo-bazel-lock.json linguist-generated=true
+ruby/extractor/cargo-bazel-lock.json -merge

.github/codeql/codeql-config.yml

@@ -9,5 +9,3 @@ paths-ignore:
   - '/python/'
   - '/javascript/ql/test'
   - '/javascript/extractor/tests'
-  - '/rust/ql/test'
-  - '/rust/ql/integration-tests'

.github/labeler.yml

@@ -30,18 +30,10 @@ Ruby:
   - ruby/**/*
   - change-notes/**/*ruby*
 
-Rust:
-  - rust/**/*
-  - change-notes/**/*rust*
-
 Swift:
   - swift/**/*
   - change-notes/**/*swift*
 
-Actions:
-  - actions/**/*
-  - change-notes/**/*actions*
-
 documentation:
   - "**/*.qhelp"
   - "**/*.md"

.github/pull_request_template.md

@@ -1,14 +0,0 @@
-### Pull Request checklist
-
-#### All query authors
-
-- [ ] A change note is added if necessary. See [the documentation](https://github.com/github/codeql/blob/main/docs/change-notes.md) in this repository.
-- [ ] All new queries have appropriate `.qhelp`. See [the documentation](https://github.com/github/codeql/blob/main/docs/query-help-style-guide.md) in this repository.
-- [ ] QL tests are added if necessary. See [Testing custom queries](https://docs.github.com/en/code-security/codeql-cli/using-the-advanced-functionality-of-the-codeql-cli/testing-custom-queries) in the GitHub documentation.
-- [ ] New and changed queries have correct query metadata. See [the documentation](https://github.com/github/codeql/blob/main/docs/query-metadata-style-guide.md) in this repository.
-
-#### Internal query authors only
-
-- [ ] Autofixes generated based on these changes are valid, only needed if this PR makes significant changes to `.ql`, `.qll`, or `.qhelp` files. See [the documentation](https://github.com/github/codeql-team/blob/main/docs/best-practices/validating-autofix-for-query-changes.md) (internal access required).
-- [ ] Changes are validated [at scale](https://github.com/github/codeql-dca/) (internal access required).
-- [ ] Adding a new query? Consider also [adding the query to autofix](https://github.com/github/codeml-autofix/blob/main/docs/updating-query-support.md#adding-a-new-query-to-the-query-suite).

.github/workflows/build-ripunzip.yml

@@ -1,74 +0,0 @@
-name: Build runzip
-
-on:
-  workflow_dispatch:
-    inputs:
-      ripunzip-version:
-        description: "what reference to checktout from google/runzip"
-        required: false
-        default: v1.2.1
-      openssl-version:
-        description: "what reference to checkout from openssl/openssl for Linux"
-        required: false
-        default: openssl-3.3.0
-
-jobs:
-  build:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-20.04, macos-13, windows-2019]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: google/ripunzip
-          ref: ${{ inputs.ripunzip-version }}
-      # we need to avoid ripunzip dynamically linking into libssl
-      # see https://github.com/sfackler/rust-openssl/issues/183
-      - if: runner.os == 'Linux'
-        name: checkout openssl
-        uses: actions/checkout@v4
-        with:
-          repository: openssl/openssl
-          path: openssl
-          ref: ${{ inputs.openssl-version }}
-      - if: runner.os == 'Linux'
-        name: build and install openssl with fPIC
-        shell: bash
-        working-directory: openssl
-        run: |
-          ./config -fPIC --prefix=$HOME/.local --openssldir=$HOME/.local/ssl
-          make -j $(nproc)
-          make install_sw -j $(nproc)
-      - if: runner.os == 'Linux'
-        name: build (linux)
-        shell: bash
-        run: |
-          env OPENSSL_LIB_DIR=$HOME/.local/lib64 OPENSSL_INCLUDE_DIR=$HOME/.local/include OPENSSL_STATIC=yes cargo build --release
-          mv target/release/ripunzip ripunzip-linux
-      - if: runner.os == 'Windows'
-        name: build (windows)
-        shell: bash
-        run: |
-          cargo build --release
-          mv target/release/ripunzip ripunzip-windows
-      - name: build (macOS)
-        if: runner.os == 'macOS'
-        shell: bash
-        run: |
-          rustup target install x86_64-apple-darwin
-          rustup target install aarch64-apple-darwin
-          cargo build --target x86_64-apple-darwin --release
-          cargo build --target aarch64-apple-darwin --release
-          lipo -create -output ripunzip-macos \
-            -arch x86_64 target/x86_64-apple-darwin/release/ripunzip \
-            -arch arm64 target/aarch64-apple-darwin/release/ripunzip
-      - uses: actions/upload-artifact@v4
-        with:
-          name: ripunzip-${{ runner.os }}
-          path: ripunzip-*
-      - name: Check built binary
-        shell: bash
-        run: |
-          ./ripunzip-* --version

.github/workflows/buildifier.yml

@@ -24,5 +24,5 @@ jobs:
           extra_args: >
             buildifier --all-files 2>&1 ||
             (
-              echo -e "In order to format all bazel files, please run:\n  bazel run //misc/bazel/buildifier"; exit 1
+              echo -e "In order to format all bazel files, please run:\n  bazel run //:buildifier"; exit 1
             )

.github/workflows/check-change-note.yml

@@ -16,12 +16,11 @@ on:
       - "shared/**/*.qll"
       - "!**/experimental/**"
       - "!ql/**"
-      - "!rust/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:
   check-change-note:
-    env:
+    env: 
       REPO: ${{ github.repository }}
       PULL_REQUEST_NUMBER: ${{ github.event.number }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -34,7 +33,7 @@ jobs:
           !contains(github.event.pull_request.labels.*.name, 'no-change-note-required')
         run: |
           change_note_files=$(gh api "repos/$REPO/pulls/$PULL_REQUEST_NUMBER/files" --paginate --jq '.[].filename | select(test("/change-notes/.*[.]md$"))')
-
+          
           if [ -z "$change_note_files" ]; then
             echo "No change note found. Either add one, or add the 'no-change-note-required' label."
             exit 1

.github/workflows/codeql-analysis.yml

@@ -56,9 +56,7 @@ jobs:
     #    uses a compiled language
 
     - run: |
-       cd csharp
-       dotnet tool restore
-       dotnet build .
+       dotnet build csharp
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@main

.github/workflows/cpp-swift-analysis.yml

@@ -37,7 +37,7 @@ jobs:
       with:
         languages: cpp
         config-file: ./.github/codeql/codeql-config.yml
-
+  
     - name: "[Ubuntu] Remove GCC 13 from runner image"
       shell: bash
       run: |
@@ -48,7 +48,7 @@ jobs:
     - name: "Build Swift extractor using Bazel"
       run: |
         bazel clean --expunge
-        bazel run //swift:create-extractor-pack --nouse_action_cache --noremote_accept_cached --noremote_upload_local_results --spawn_strategy=local
+        bazel run //swift:create-extractor-pack --nouse_action_cache --noremote_accept_cached --noremote_upload_local_results --spawn_strategy=local --features=-layering_check
         bazel shutdown
 
     - name: Perform CodeQL Analysis

.github/workflows/csharp-qltest.yml

@@ -29,6 +29,45 @@ permissions:
   contents: read
 
 jobs:
+  qlupgrade:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/fetch-codeql
+      - name: Check DB upgrade scripts
+        run: |
+          echo >empty.trap
+          codeql dataset import -S ql/lib/upgrades/initial/semmlecode.csharp.dbscheme testdb empty.trap
+          codeql dataset upgrade testdb --additional-packs ql/lib
+          diff -q testdb/semmlecode.csharp.dbscheme ql/lib/semmlecode.csharp.dbscheme
+      - name: Check DB downgrade scripts
+        run: |
+          echo >empty.trap
+          rm -rf testdb; codeql dataset import -S ql/lib/semmlecode.csharp.dbscheme testdb empty.trap
+          codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
+           --dbscheme=ql/lib/semmlecode.csharp.dbscheme --target-dbscheme=downgrades/initial/semmlecode.csharp.dbscheme |
+           xargs codeql execute upgrades testdb
+          diff -q testdb/semmlecode.csharp.dbscheme downgrades/initial/semmlecode.csharp.dbscheme
+  qltest:
+    if: github.repository_owner == 'github'
+    runs-on: ubuntu-latest-xl
+    strategy:
+      fail-fast: false
+      matrix:
+        slice: ["1/2", "2/2"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./csharp/actions/create-extractor-pack
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: csharp-qltest-${{ matrix.slice }}
+      - name: Run QL tests
+        run: |
+          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
   unit-tests:
     strategy:
       matrix:
@@ -42,11 +81,10 @@ jobs:
           dotnet-version: 8.0.101
       - name: Extractor unit tests
         run: |
-          dotnet tool restore
           dotnet test -p:RuntimeFrameworkVersion=8.0.1 extractor/Semmle.Util.Tests
           dotnet test -p:RuntimeFrameworkVersion=8.0.1 extractor/Semmle.Extraction.Tests
           dotnet test -p:RuntimeFrameworkVersion=8.0.1 autobuilder/Semmle.Autobuild.CSharp.Tests
-          dotnet test -p:RuntimeFrameworkVersion=8.0.1 autobuilder/Semmle.Autobuild.Cpp.Tests
+          dotnet test -p:RuntimeFrameworkVersion=8.0.1 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"
         shell: bash
   stubgentest:
     runs-on: ubuntu-latest
@@ -62,6 +100,6 @@ jobs:
           # Update existing stubs in the repo with the freshly generated ones
           mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/
           git status
-          codeql test run --threads=0 --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
+          codeql test run --threads=0 --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/csv-coverage-metrics.yml

@@ -37,7 +37,7 @@ jobs:
         run: |
           DATABASE="${{ runner.temp }}/java-database"
           codeql database analyze --format=sarif-latest --output=metrics-java.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: metrics-java.sarif
           path: metrics-java.sarif
@@ -64,7 +64,7 @@ jobs:
         run: |
           DATABASE="${{ runner.temp }}/csharp-database"
           codeql database analyze --format=sarif-latest --output=metrics-csharp.sarif -- "$DATABASE" ./csharp/ql/src/Metrics/Summaries/FrameworkCoverage.ql
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: metrics-csharp.sarif
           path: metrics-csharp.sarif

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -71,21 +71,21 @@ jobs:
         run: |
           python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
       - name: Upload CSV package list
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: csv-framework-coverage-merge
           path: |
             out_merge/framework-coverage-*.csv
             out_merge/framework-coverage-*.rst
       - name: Upload CSV package list
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: csv-framework-coverage-base
           path: |
             out_base/framework-coverage-*.csv
             out_base/framework-coverage-*.rst
       - name: Upload comparison results
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: comparison
           path: |
@@ -97,7 +97,7 @@ jobs:
         env:
           PR_NUMBER: ${{ github.event.pull_request.number }}
       - name: Upload PR number
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: pr
           path: pr/
@@ -117,7 +117,7 @@ jobs:
           GITHUB_TOKEN: ${{ github.token }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
       - name: Upload comment ID (if it exists)
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: comment
           path: comment/

.github/workflows/csv-coverage-timeseries.yml

@@ -30,7 +30,7 @@ jobs:
         run: |
           python script/misc/scripts/library-coverage/generate-timeseries.py codeqlModels
       - name: Upload timeseries CSV
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: framework-coverage-timeseries
           path: framework-coverage-timeseries-*.csv

.github/workflows/csv-coverage.yml

@@ -34,12 +34,12 @@ jobs:
         run: |
           python script/misc/scripts/library-coverage/generate-report.py ci codeqlModels script
       - name: Upload CSV package list
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: framework-coverage-csv
           path: framework-coverage-*.csv
       - name: Upload RST package list
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: framework-coverage-rst
           path: framework-coverage-*.rst

.github/workflows/go-tests-other-os.yml

@@ -7,9 +7,8 @@ on:
       - .github/workflows/go-tests-other-os.yml
       - .github/actions/**
       - codeql-workspace.yml
-      - MODULE.bazel
-      - .bazelrc
-      - misc/bazel/**
+env:
+  GO_VERSION: '~1.22.0'
 
 permissions:
   contents: read
@@ -19,17 +18,72 @@ jobs:
     name: Test MacOS
     runs-on: macos-latest
     steps:
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+        id: go
+
       - name: Check out code
         uses: actions/checkout@v4
-      - name: Run tests
-        uses: ./go/actions/test
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: go-qltest
+      - name: Test
+        run: |
+          cd go
+          make test cache="${{ steps.query-cache.outputs.cache-dir }}"
 
   test-win:
     if: github.repository_owner == 'github'
     name: Test Windows
     runs-on: windows-latest-xl
     steps:
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+        id: go
+
       - name: Check out code
         uses: actions/checkout@v4
-      - name: Run tests
-        uses: ./go/actions/test
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: go-qltest
+
+      - name: Test
+        run: |
+          cd go
+          make test cache="${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/go-tests.yml

@@ -3,7 +3,6 @@ on:
   push:
     paths:
       - "go/**"
-      - "shared/**"
       - .github/workflows/go-tests.yml
       - .github/actions/**
       - codeql-workspace.yml
@@ -13,13 +12,12 @@ on:
   pull_request:
     paths:
       - "go/**"
-      - "shared/**"
       - .github/workflows/go-tests.yml
       - .github/actions/**
       - codeql-workspace.yml
-      - MODULE.bazel
-      - .bazelrc
-      - misc/bazel/**
+
+env:
+  GO_VERSION: '~1.22.0'
 
 permissions:
   contents: read
@@ -30,9 +28,51 @@ jobs:
     name: Test Linux (Ubuntu)
     runs-on: ubuntu-latest-xl
     steps:
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+        id: go
+
       - name: Check out code
         uses: actions/checkout@v4
-      - name: Run tests
-        uses: ./go/actions/test
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Check that all Go code is autoformatted
+        run: |
+          cd go
+          make check-formatting
+
+      - name: Compile qhelp files to markdown
+        run: |
+          cd go
+          env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown
+
+      - name: Upload qhelp markdown
+        uses: actions/upload-artifact@v3
         with:
-          run-code-checks: true
+          name: qhelp-markdown
+          path: go/qhelp-out/**/*.md
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: go-qltest
+
+      - name: Test
+        run: |
+          cd go
+          make test cache="${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/kotlin-build.yml

@@ -1,28 +0,0 @@
-name: "Kotlin Build"
-
-on:
-  pull_request:
-    paths:
-      - "java/kotlin-extractor/**"
-      - "misc/bazel/**"
-      - "misc/codegen/**"
-      - "*.bazel*"
-      - .github/workflows/kotlin-build.yml
-    branches:
-      - main
-      - rc/*
-      - codeql-cli-*
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - run: |
-          bazel query //java/kotlin-extractor/...
-          # only build the default version as a quick check that we can build from `codeql`
-          # the full official build will be checked by QLucie
-          bazel build //java/kotlin-extractor

.github/workflows/mad_modelDiff.yml

@@ -38,20 +38,14 @@ jobs:
           path: codeql-main
           ref: main
       - uses: ./codeql-main/.github/actions/fetch-codeql
-      # compute the shortname of the project that does not contain any special (disk) characters
-      - run: |
-          echo "SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}" >> $GITHUB_OUTPUT
-        env: 
-          SLUG: ${{ matrix.slug }}
-        id: shortname
       - name: Download database
         env:
           SLUG: ${{ matrix.slug }}
           GH_TOKEN: ${{ github.token }}
-          SHORTNAME: ${{ steps.shortname.outputs.SHORTNAME }}
         run: |
           set -x
           mkdir lib-dbs
+          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
           gh api -H "Accept: application/zip" "/repos/${SLUG}/code-scanning/codeql/databases/java" > "$SHORTNAME.zip"
           unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
           mkdir "lib-dbs/$SHORTNAME/"
@@ -99,14 +93,14 @@ jobs:
             name="diff_${basename/.model.yml/""}"
             (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
           done
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
-          name: models-${{ steps.shortname.outputs.SHORTNAME }}
+          name: models
           path: tmp-models/**/**/*.model.yml
           retention-days: 20
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
-          name: diffs-${{ steps.shortname.outputs.SHORTNAME }}
+          name: diffs
           path: tmp-models/*.html
           # An html file is only produced if the generated models differ.
           if-no-files-found: ignore

.github/workflows/mad_regenerate-models.yml

@@ -59,7 +59,7 @@ jobs:
           find java -name "*.model.yml" -print0 | xargs -0 git add
           git status
           git diff --cached > models.patch
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: patch
           path: models.patch

.github/workflows/post-pr-comment.yml

@@ -17,11 +17,8 @@ jobs:
   post_comment:
     runs-on: ubuntu-latest
     steps:
-      - name: Download artifacts
-        run: |
-          gh run download "${WORKFLOW_RUN_ID}" --repo "${GITHUB_REPOSITORY}" --name "comment-pr-number"
-          gh run download "${WORKFLOW_RUN_ID}" --repo "${GITHUB_REPOSITORY}" --name "comment-body"
-          gh run download "${WORKFLOW_RUN_ID}" --repo "${GITHUB_REPOSITORY}" --name "comment-id"
+      - name: Download artifact
+        run: gh run download "${WORKFLOW_RUN_ID}" --repo "${GITHUB_REPOSITORY}" --name "comment"
         env:
           GITHUB_TOKEN: ${{ github.token }}
           WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}

.github/workflows/qhelp-pr-preview.yml

@@ -36,9 +36,9 @@ jobs:
       - run: echo "${PR_NUMBER}" > pr_number.txt
         env:
           PR_NUMBER: ${{ github.event.number }}
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
-          name: comment-pr-number
+          name: comment
           path: pr_number.txt
           if-no-files-found: error
           retention-days: 1
@@ -78,9 +78,9 @@ jobs:
           exit "${EXIT_CODE}"
 
       - if: ${{ !cancelled() }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: comment-body
+          name: comment
           path: comment_body.txt
           if-no-files-found: error
           retention-days: 1
@@ -94,9 +94,9 @@ jobs:
           GITHUB_TOKEN: ${{ github.token }}
           PR_NUMBER: ${{ github.event.number }}
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
-          name: comment-id
+          name: comment
           path: comment_id.txt
           if-no-files-found: error
           retention-days: 1

.github/workflows/ql-for-ql-build.yml

@@ -49,20 +49,20 @@ jobs:
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Release build
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; ./scripts/create-extractor-pack.sh
+        run: cd ql; ./scripts/create-extractor-pack.sh       
         env:
-          GH_TOKEN: ${{ github.token }}
+          GH_TOKEN: ${{ github.token }}   
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with:
+        with: 
           key: run-ql-for-ql
       - name: Make database and analyze
         run: |
           ./ql/target/release/buramu | tee deprecated.blame # Add a blame file for the extractor to parse.
-          ${CODEQL} database create -l=ql ${DB} --search-path "${{ github.workspace }}"
+          ${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB}
           ${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls  --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-        env:
+        env: 
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
           DB: ${{ runner.temp }}/DB
           LGTM_INDEX_FILTERS: |
@@ -75,7 +75,7 @@ jobs:
           sarif_file: ql-for-ql.sarif
           category: ql-for-ql
       - name: Sarif as artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: ql-for-ql.sarif
           path: ql-for-ql.sarif
@@ -84,7 +84,7 @@ jobs:
           mkdir split-sarif
           node ./ql/scripts/split-sarif.js ql-for-ql.sarif split-sarif
       - name: Upload langs as artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: ql-for-ql-langs
           path: split-sarif

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -53,8 +53,8 @@ jobs:
       - name: Create database
         run: |
           "${CODEQL}" database create \
-          --search-path "${{ github.workspace }}"
-          --threads 4 \
+            --search-path "ql/extractor-pack" \
+            --threads 4 \
             --language ql --source-root "${{ github.workspace }}/repo" \
             "${{ runner.temp }}/database"
         env:
@@ -65,7 +65,7 @@ jobs:
           "${CODEQL}" dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ql"
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: measurements
           path: stats
@@ -76,14 +76,14 @@ jobs:
     needs: measure
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v3
         with:
           name: measurements
           path: stats
       - run: |
           python -m pip install --user lxml
           find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ruby/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: ql.dbscheme.stats
           path: ql/ql/src/ql.dbscheme.stats

.github/workflows/ql-for-ql-tests.yml

@@ -49,15 +49,15 @@ jobs:
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with:
+        with: 
           key: ql-for-ql-tests
       - name: Run QL tests
         run: |
-          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
+          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
 
-  other-os:
+  other-os: 
     strategy:
       matrix:
         os: [macos-latest, windows-latest]
@@ -65,7 +65,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
-      - name: Install GNU tar
+      - name: Install GNU tar 
         if: runner.os == 'macOS'
         run: |
           brew install gnu-tar
@@ -100,7 +100,7 @@ jobs:
       - name: Run a single QL tests - Unix
         if: runner.os != 'Windows'
         run: |
-          "${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
+          "${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Run a single QL tests - Windows
@@ -108,4 +108,5 @@ jobs:
         shell: pwsh
         run: |
           $Env:PATH += ";$(dirname ${{ steps.find-codeql.outputs.codeql-path }})"
-          codeql test run --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
+          codeql test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
+      

.github/workflows/query-list.yml

@@ -37,7 +37,7 @@ jobs:
       run: |
         python codeql/misc/scripts/generate-code-scanning-query-list.py > code-scanning-query-list.csv
     - name: Upload code scanning query list
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v3
       with:
         name: code-scanning-query-list
         path: code-scanning-query-list.csv

.github/workflows/ruby-build.yml

@@ -7,7 +7,6 @@ on:
       - .github/workflows/ruby-build.yml
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
-      - "shared/tree-sitter-extractor/**"
     branches:
       - main
       - "rc/*"
@@ -17,7 +16,6 @@ on:
       - .github/workflows/ruby-build.yml
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
-      - "shared/tree-sitter-extractor/**"
     branches:
       - main
       - "rc/*"
@@ -65,8 +63,8 @@ jobs:
         id: cache-extractor
         with:
           path: |
-            target/release/codeql-extractor-ruby
-            target/release/codeql-extractor-ruby.exe
+            ruby/extractor/target/release/codeql-extractor-ruby
+            ruby/extractor/target/release/codeql-extractor-ruby.exe
             ruby/extractor/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ruby/extractor/**/*.rs') }}
       - uses: actions/cache@v3
@@ -75,7 +73,7 @@ jobs:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
-            target
+            ruby/target
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-rust-cargo-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/**/Cargo.lock') }}
       - name: Check formatting
         if: steps.cache-extractor.outputs.cache-hit != 'true'
@@ -91,23 +89,23 @@ jobs:
         run: cd extractor && cargo build --release
       - name: Generate dbscheme
         if: ${{ matrix.os == 'ubuntu-latest' && steps.cache-extractor.outputs.cache-hit != 'true'}}
-        run: ../target/release/codeql-extractor-ruby generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v4
+        run: extractor/target/release/codeql-extractor-ruby generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+      - uses: actions/upload-artifact@v3
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           name: ruby.dbscheme
           path: ruby/ql/lib/ruby.dbscheme
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           name: TreeSitter.qll
           path: ruby/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: extractor-${{ matrix.os }}
           path: |
-            target/release/codeql-extractor-ruby
-            target/release/codeql-extractor-ruby.exe
+            ruby/extractor/target/release/codeql-extractor-ruby
+            ruby/extractor/target/release/codeql-extractor-ruby.exe
           retention-days: 1
   compile-queries:
     if: github.repository_owner == 'github'
@@ -134,32 +132,31 @@ jobs:
           PACK_FOLDER=$(readlink -f "$PACKS"/codeql/ruby-queries/*)
           codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
           (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: codeql-ruby-queries
           path: |
             ${{ runner.temp }}/query-packs/*
           retention-days: 1
-          include-hidden-files: true
 
   package:
     runs-on: ubuntu-latest
     needs: [build, compile-queries]
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v3
         with:
           name: ruby.dbscheme
           path: ruby/ruby
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v3
         with:
           name: extractor-ubuntu-latest
           path: ruby/linux64
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v3
         with:
           name: extractor-windows-latest
           path: ruby/win64
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v3
         with:
           name: extractor-macos-latest
           path: ruby/osx64
@@ -172,13 +169,12 @@ jobs:
           cp win64/codeql-extractor-ruby.exe ruby/tools/win64/extractor.exe
           chmod +x ruby/tools/{linux64,osx64}/extractor
           zip -rq codeql-ruby.zip ruby
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: codeql-ruby-pack
           path: ruby/codeql-ruby.zip
           retention-days: 1
-          include-hidden-files: true
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v3
         with:
           name: codeql-ruby-queries
           path: ruby/qlpacks
@@ -190,12 +186,11 @@ jobs:
             ]
           }' > .codeqlmanifest.json
           zip -rq codeql-ruby-bundle.zip .codeqlmanifest.json ruby qlpacks
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: codeql-ruby-bundle
           path: ruby/codeql-ruby-bundle.zip
           retention-days: 1
-          include-hidden-files: true
 
   test:
     defaults:
@@ -214,7 +209,7 @@ jobs:
         uses: ./.github/actions/fetch-codeql
 
       - name: Download Ruby bundle
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3
         with:
           name: codeql-ruby-bundle
           path: ${{ runner.temp }}

.github/workflows/ruby-dataset-measure.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Create database
         run: |
           codeql database create \
-            --search-path "${{ github.workspace }}" \
+            --search-path "${{ github.workspace }}/ruby/extractor-pack" \
             --threads 4 \
             --language ruby --source-root "${{ github.workspace }}/repo" \
             "${{ runner.temp }}/database"
@@ -52,9 +52,9 @@ jobs:
         run: |
           mkdir -p "stats/${{ matrix.repo }}"
           codeql dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ruby"
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
-          name: measurements-${{ hashFiles('stats/**') }}
+          name: measurements
           path: stats
           retention-days: 1
 
@@ -63,13 +63,14 @@ jobs:
     needs: measure
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v3
         with:
+          name: measurements
           path: stats
       - run: |
           python -m pip install --user lxml
           find stats -name 'stats.xml' | sort | xargs python ruby/scripts/merge_stats.py --output ruby/ql/lib/ruby.dbscheme.stats --normalise ruby_tokeninfo
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: ruby.dbscheme.stats
           path: ruby/ql/lib/ruby.dbscheme.stats

.github/workflows/ruby-qltest.yml

@@ -64,10 +64,10 @@ jobs:
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with:
+        with: 
           key: ruby-qltest
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/rust-analysis.yml

@@ -1,64 +0,0 @@
-name: "Code scanning - Rust"
-
-on:
-  push:
-    branches:
-      - main
-      - 'rc/*'
-  pull_request:
-    branches:
-      - main
-      - 'rc/*'
-    paths:
-      - '**/*.rs'
-      - '**/Cargo.toml'
-      - '.github/codeql/codeql-config.yml'
-      - '.github/workflows/rust-analysis.yml'
-  schedule:
-    - cron: '0 9 * * 1'
-
-env:
-  CODEQL_ENABLE_EXPERIMENTAL_FEATURES: "true"
-
-jobs:
-  analyze:
-    strategy:
-      matrix:
-        language: [ 'rust' ]
-
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: read
-      security-events: write
-      pull-requests: read
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    - name: Query latest nightly CodeQL bundle
-      shell: bash
-      id: codeql
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-      run: |
-        REPO=dsp-testing/codeql-cli-nightlies
-        TAG=$(
-          gh release list -R $REPO -L1 --exclude-drafts --json tagName -q ".[] | .tagName"
-        )
-        echo "nightly_bundle=https://github.com/$REPO/releases/download/$TAG/codeql-bundle-linux64.tar.zst" \
-          | tee -a "$GITHUB_OUTPUT"
-
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@main
-      with:
-        tools: ${{ steps.codeql.outputs.nightly_bundle }}
-        languages: ${{ matrix.language }}
-        config-file: ./.github/codeql/codeql-config.yml
-
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@main
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@main

.github/workflows/rust.yml

@@ -1,58 +0,0 @@
-name: "Rust"
-
-on:
-  pull_request:
-    paths:
-      - "rust/**"
-      - "misc/bazel/**"
-      - "misc/codegen/**"
-      - "shared/**"
-      - "MODULE.bazel"
-      - .github/workflows/rust.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-      - "!**/*.md"
-      - "!**/*.qhelp"
-    branches:
-      - rust-experiment
-      - main
-      - rc/*
-      - codeql-cli-*
-
-permissions:
-  contents: read
-
-jobs:
-  rust-code:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Format
-        working-directory: rust/extractor
-        shell: bash
-        run: |
-          cargo fmt --check
-      - name: Compilation
-        working-directory: rust/extractor
-        shell: bash
-        run: cargo check
-      - name: Clippy
-        working-directory: rust/extractor
-        shell: bash
-        run: |
-          cargo clippy --fix
-          git diff --exit-code
-  rust-codegen:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install CodeQL
-        uses: ./.github/actions/fetch-codeql
-      - name: Code generation
-        shell: bash
-        run: |
-          bazel run //rust/codegen
-          git add .
-          git diff --exit-code HEAD

.github/workflows/swift.yml

@@ -44,7 +44,7 @@ jobs:
   # without waiting for the macOS build
   build-and-test-macos:
     if: github.repository_owner == 'github'
-    runs-on: macos-13-xlarge
+    runs-on: macos-12-xl
     steps:
       - uses: actions/checkout@v4
       - uses: ./swift/actions/build-and-test
@@ -64,10 +64,25 @@ jobs:
   qltests-macos:
     if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }}
     needs: build-and-test-macos
-    runs-on: macos-13-xlarge
+    runs-on: macos-12-xl
     steps:
       - uses: actions/checkout@v4
       - uses: ./swift/actions/run-ql-tests
+  integration-tests-linux:
+    if: github.repository_owner == 'github'
+    needs: build-and-test-linux
+    runs-on: ubuntu-latest-xl
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./swift/actions/run-integration-tests
+  integration-tests-macos:
+    if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }}
+    needs: build-and-test-macos
+    runs-on: macos-12-xl
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./swift/actions/run-integration-tests
   clang-format:
     if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
@@ -98,7 +113,7 @@ jobs:
       - name: Generate C++ files
         run: |
           bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/generated-cpp-files
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: swift-generated-cpp-files
           path: generated-cpp-files/**

.github/workflows/zipmerge-test.yml

@@ -1,23 +0,0 @@
-name: "Test zipmerge code"
-
-on:
-  pull_request:
-    paths:
-      - "misc/bazel/internal/zipmerge/**"
-      - "MODULE.bazel"
-      - ".bazelrc*"
-    branches:
-      - main
-      - "rc/*"
-
-permissions:
-  contents: read
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-      - run: |
-          bazel test //misc/bazel/internal/zipmerge:test --test_output=all

.gitignore

@@ -7,8 +7,8 @@
 .cache
 
 # qltest projects and artifacts
-*.actual
 */ql/test/**/*.testproj
+*/ql/test/**/*.actual
 */ql/test/**/go.sum
 
 # Visual studio temporaries, except a file used by QL4VS
@@ -62,12 +62,3 @@ node_modules/
 
 # Temporary folders for working with generated models
 .model-temp
-
-# bazel-built in-tree extractor packs
-/*/extractor-pack
-
-# Jetbrains IDE files
-.idea
-
-# cargo build directory
-/target

.lfsconfig

@@ -1,7 +0,0 @@
-[lfs]
-# codeql is publicly forked by many users, and we don't want any LFS file polluting their working
-# copies. We therefore exclude everything by default.
-# For files required by bazel builds, use rules in `misc/bazel/lfs.bzl` to download them on demand.
-# we go for `fetchinclude` to something not exsiting rather than `fetchexclude = *` because the
-# former is easier to override (with `git -c` or a local git config) to fetch something specific
-fetchinclude = /nothing

.pre-commit-config.yaml

@@ -5,9 +5,9 @@ repos:
     rev: v3.2.0
     hooks:
       - id: trailing-whitespace
-        exclude: /test/.*$(?<!\.qlref)|.*\.patch$|.*\.qll?$
+        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)|.*\.patch
       - id: end-of-file-fixer
-        exclude: /test/.*$(?<!\.qlref)|.*\.patch$|.*\.qll?$
+        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)|.*\.patch
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: v17.0.6
@@ -15,7 +15,7 @@ repos:
       - id: clang-format
 
   - repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v2.0.4
+    rev: v1.6.0
     hooks:
       - id: autopep8
         files: ^misc/codegen/.*\.py
@@ -26,17 +26,9 @@ repos:
         name: Format bazel files
         files: \.(bazel|bzl)
         language: system
-        entry: bazel run //misc/bazel/buildifier
+        entry: bazel run //:buildifier
         pass_filenames: false
 
-#      DISABLED: can be enabled by copying this config and installing `pre-commit` with `--config` on the copy
-#      - id: go-gen
-#        name: Check checked in generated files in go
-#        files: ^go/.*
-#        language: system
-#        entry: bazel run //go:gen
-#        pass_filenames: false
-
       - id: codeql-format
         name: Fix QL file formatting
         files: \.qll?$
@@ -45,7 +37,7 @@ repos:
 
       - id: sync-files
         name: Fix files required to be identical
-        files: \.(qll?|qhelp|swift|toml)$|^config/identical-files\.json$
+        files: \.(qll?|qhelp|swift)$|^config/identical-files\.json$
         language: system
         entry: python3 config/sync-files.py --latest
         pass_filenames: false
@@ -58,7 +50,7 @@ repos:
 
       - id: swift-codegen
         name: Run Swift checked in code generation
-        files: ^misc/codegen/|^swift/(schema.py$|codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements)|ql/\.generated.list)
+        files: ^swift/(schema.py$|codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements)|ql/\.generated.list)
         language: system
         entry: bazel run //swift/codegen -- --quiet
         pass_filenames: false
@@ -69,17 +61,3 @@ repos:
         language: system
         entry: bazel test //misc/codegen/test
         pass_filenames: false
-
-      - id: rust-codegen
-        name: Run Rust checked in code generation
-        files: ^misc/codegen/|^rust/(schema.py$|codegen/|.*/generated/|ql/lib/(rust\.dbscheme$|codeql/rust/elements)|\.generated.list)
-        language: system
-        entry: bazel run //rust/codegen -- --quiet
-        pass_filenames: false
-
-      - id: rust-lint
-        name: Run fmt and clippy on Rust code
-        files: ^rust/extractor/(.*rs|Cargo.toml)$
-        language: system
-        entry: python3 rust/lint.py
-        pass_filenames: false

.vscode/settings.json

@@ -1,6 +1,5 @@
 {
   "omnisharp.autoStart": false,
   "cmake.sourceDirectory": "${workspaceFolder}/swift",
-  "cmake.buildDirectory": "${workspaceFolder}/bazel-cmake-build",
-  "editor.suggest.matchOnWordStartOnly": false
+  "cmake.buildDirectory": "${workspaceFolder}/bazel-cmake-build"
 }

BUILD.bazel

@@ -1 +1,9 @@
-exports_files(["LICENSE"])
+load("@buildifier_prebuilt//:rules.bzl", "buildifier")
+
+buildifier(
+    name = "buildifier",
+    exclude_patterns = [
+        "./.git/*",
+    ],
+    lint_mode = "fix",
+)

CODEOWNERS

@@ -1,7 +1,6 @@
 /cpp/ @github/codeql-c-analysis
+/cpp/autobuilder/ @github/codeql-c-extractor
 /csharp/ @github/codeql-csharp
-/csharp/autobuilder/Semmle.Autobuild.Cpp @github/codeql-c-extractor
-/csharp/autobuilder/Semmle.Autobuild.Cpp.Tests @github/codeql-c-extractor
 /go/ @github/codeql-go
 /java/ @github/codeql-java
 /javascript/ @github/codeql-javascript
@@ -23,6 +22,7 @@
 /ql/ @github/codeql-ql-for-ql-reviewers
 
 # Bazel (excluding BUILD.bazel files)
+WORKSPACE.bazel @github/codeql-ci-reviewers
 MODULE.bazel @github/codeql-ci-reviewers
 .bazelversion @github/codeql-ci-reviewers
 .bazelrc @github/codeql-ci-reviewers

CONTRIBUTING.md

@@ -4,8 +4,6 @@ We welcome contributions to our CodeQL libraries and queries. Got an idea for a
 
 There is lots of useful documentation to help you write queries, ranging from information about query file structure to tutorials for specific target languages. For more information on the documentation available, see [CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/codeql-queries) on [codeql.github.com](https://codeql.github.com).
 
-Note that the CodeQL for Visual Studio Code documentation has been migrated to https://docs.github.com/en/code-security/codeql-for-vs-code/, but you can still contribute to it via a different repository. For more information, see [Contributing to GitHub Docs documentation](https://docs.github.com/en/contributing)."
-
 ## Change notes
 
 Any nontrivial user-visible change to a query pack or library pack should have a change note. For details on how to add a change note for your change, see [this guide](docs/change-notes.md).
@@ -45,7 +43,7 @@ If you have an idea for a query that you would like to share with other CodeQL u
 
 3. **Formatting**
 
-    - The queries and libraries must be autoformatted, for example using the "Format Document" command in [CodeQL for Visual Studio Code](https://docs.github.com/en/code-security/codeql-for-vs-code/).
+    - The queries and libraries must be autoformatted, for example using the "Format Document" command in [CodeQL for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-codeql-for-visual-studio-code).
 
     If you prefer, you can either:
     1. install the [pre-commit framework](https://pre-commit.com/) and install the configured hooks on this repo via `pre-commit install`, or

Cargo.toml

@@ -1,16 +0,0 @@
-# This is the shared workspace file for extractor using shared/tree-sitter/extractor
-[workspace]
-
-resolver = "2"
-members = [
-    "shared/tree-sitter-extractor",
-    "ruby/extractor",
-    "rust/extractor",
-    "rust/extractor/macros",
-    "rust/ast-generator",
-]
-
-[patch.crates-io]
-# patch for build script bug preventing bazel build
-# see https://github.com/rust-lang/rustc_apfloat/pull/17
-rustc_apfloat = { git = "https://github.com/redsun82/rustc_apfloat.git", rev = "096d585100636bc2e9f09d7eefec38c5b334d47b" }

MODULE.bazel

@@ -1,7 +1,6 @@
 module(
-    name = "ql",
+    name = "codeql",
     version = "0.0",
-    repo_name = "codeql",
 )
 
 # this points to our internal repository when `codeql` is checked out as a submodule thereof
@@ -14,68 +13,17 @@ local_path_override(
 
 # see https://registry.bazel.build/ for a list of available packages
 
-bazel_dep(name = "platforms", version = "0.0.10")
-bazel_dep(name = "rules_go", version = "0.50.0")
-bazel_dep(name = "rules_pkg", version = "1.0.1")
-bazel_dep(name = "rules_nodejs", version = "6.2.0-codeql.1")
-bazel_dep(name = "rules_python", version = "0.36.0")
-bazel_dep(name = "bazel_skylib", version = "1.7.1")
+bazel_dep(name = "platforms", version = "0.0.8")
+bazel_dep(name = "rules_pkg", version = "0.10.1")
+bazel_dep(name = "rules_nodejs", version = "6.0.3")
+bazel_dep(name = "rules_python", version = "0.31.0")
+bazel_dep(name = "bazel_skylib", version = "1.5.0")
 bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl")
 bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
 bazel_dep(name = "fmt", version = "10.0.0")
-bazel_dep(name = "rules_kotlin", version = "2.0.0-codeql.1")
-bazel_dep(name = "gazelle", version = "0.38.0")
-bazel_dep(name = "rules_dotnet", version = "0.16.1")
-bazel_dep(name = "googletest", version = "1.14.0.bcr.1")
-bazel_dep(name = "rules_rust", version = "0.52.2")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
-# crate_py but shortened due to Windows file path considerations
-cp = use_extension(
-    "@rules_rust//crate_universe:extension.bzl",
-    "crate",
-    isolate = True,
-)
-cp.from_cargo(
-    name = "py_deps",
-    cargo_lockfile = "//python/extractor/tsg-python:Cargo.lock",
-    manifests = [
-        "//python/extractor/tsg-python:Cargo.toml",
-        "//python/extractor/tsg-python/tsp:Cargo.toml",
-    ],
-)
-use_repo(cp, "py_deps")
-
-# deps for ruby+rust, but shortened due to windows file paths
-r = use_extension(
-    "@rules_rust//crate_universe:extension.bzl",
-    "crate",
-    isolate = True,
-)
-r.from_cargo(
-    name = "r",
-    cargo_lockfile = "//:Cargo.lock",
-    manifests = [
-        "//:Cargo.toml",
-        "//ruby/extractor:Cargo.toml",
-        "//rust/extractor:Cargo.toml",
-        "//rust/extractor/macros:Cargo.toml",
-        "//rust/ast-generator:Cargo.toml",
-        "//shared/tree-sitter-extractor:Cargo.toml",
-    ],
-)
-use_repo(r, tree_sitter_extractors_deps = "r")
-
-dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet")
-dotnet.toolchain(dotnet_version = "8.0.101")
-use_repo(dotnet, "dotnet_toolchains")
-
-register_toolchains("@dotnet_toolchains//:all")
-
-csharp_main_extension = use_extension("//csharp:paket.main_extension.bzl", "main_extension")
-use_repo(csharp_main_extension, "paket.main")
-
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 pip.parse(
     hub_name = "codegen_deps",
@@ -100,102 +48,10 @@ use_repo(
 node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node")
 node.toolchain(
     name = "nodejs",
-    node_urls = [
-        "https://nodejs.org/dist/v{version}/{filename}",
-        "https://mirrors.dotsrc.org/nodejs/release/v{version}/{filename}",
-    ],
     node_version = "18.15.0",
 )
 use_repo(node, "nodejs", "nodejs_toolchains")
 
-kotlin_extractor_deps = use_extension("//java/kotlin-extractor:deps.bzl", "kotlin_extractor_deps")
-
-# following list can be kept in sync by running `bazel mod tidy` in `codeql`
-use_repo(
-    kotlin_extractor_deps,
-    "codeql_kotlin_defaults",
-    "codeql_kotlin_embeddable",
-    "kotlin-compiler-1.5.0",
-    "kotlin-compiler-1.5.10",
-    "kotlin-compiler-1.5.20",
-    "kotlin-compiler-1.5.30",
-    "kotlin-compiler-1.6.0",
-    "kotlin-compiler-1.6.20",
-    "kotlin-compiler-1.7.0",
-    "kotlin-compiler-1.7.20",
-    "kotlin-compiler-1.8.0",
-    "kotlin-compiler-1.9.0-Beta",
-    "kotlin-compiler-1.9.20-Beta",
-    "kotlin-compiler-2.0.0-RC1",
-    "kotlin-compiler-2.0.20-Beta2",
-    "kotlin-compiler-2.1.0-Beta1",
-    "kotlin-compiler-embeddable-1.5.0",
-    "kotlin-compiler-embeddable-1.5.10",
-    "kotlin-compiler-embeddable-1.5.20",
-    "kotlin-compiler-embeddable-1.5.30",
-    "kotlin-compiler-embeddable-1.6.0",
-    "kotlin-compiler-embeddable-1.6.20",
-    "kotlin-compiler-embeddable-1.7.0",
-    "kotlin-compiler-embeddable-1.7.20",
-    "kotlin-compiler-embeddable-1.8.0",
-    "kotlin-compiler-embeddable-1.9.0-Beta",
-    "kotlin-compiler-embeddable-1.9.20-Beta",
-    "kotlin-compiler-embeddable-2.0.0-RC1",
-    "kotlin-compiler-embeddable-2.0.20-Beta2",
-    "kotlin-compiler-embeddable-2.1.0-Beta1",
-    "kotlin-stdlib-1.5.0",
-    "kotlin-stdlib-1.5.10",
-    "kotlin-stdlib-1.5.20",
-    "kotlin-stdlib-1.5.30",
-    "kotlin-stdlib-1.6.0",
-    "kotlin-stdlib-1.6.20",
-    "kotlin-stdlib-1.7.0",
-    "kotlin-stdlib-1.7.20",
-    "kotlin-stdlib-1.8.0",
-    "kotlin-stdlib-1.9.0-Beta",
-    "kotlin-stdlib-1.9.20-Beta",
-    "kotlin-stdlib-2.0.0-RC1",
-    "kotlin-stdlib-2.0.20-Beta2",
-    "kotlin-stdlib-2.1.0-Beta1",
-)
-
-go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
-go_sdk.download(version = "1.23.1")
-
-go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
-go_deps.from_file(go_mod = "//go/extractor:go.mod")
-use_repo(go_deps, "org_golang_x_mod", "org_golang_x_tools")
-
-lfs_files = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_files")
-
-lfs_files(
-    name = "ripunzip-linux",
-    srcs = ["//misc/ripunzip:ripunzip-linux"],
-    executable = True,
-)
-
-lfs_files(
-    name = "ripunzip-windows",
-    srcs = ["//misc/ripunzip:ripunzip-windows.exe"],
-    executable = True,
-)
-
-lfs_files(
-    name = "ripunzip-macos",
-    srcs = ["//misc/ripunzip:ripunzip-macos"],
-    executable = True,
-)
-
-lfs_files(
-    name = "swift-resource-dir-linux",
-    srcs = ["//swift/third_party/resource-dir:resource-dir-linux.zip"],
-)
-
-lfs_files(
-    name = "swift-resource-dir-macos",
-    srcs = ["//swift/third_party/resource-dir:resource-dir-macos.zip"],
-)
-
 register_toolchains(
     "@nodejs_toolchains//:all",
 )

README.md

@@ -4,7 +4,7 @@ This open source repository contains the standard CodeQL libraries and queries t
 
 ## How do I learn CodeQL and run queries?
 
-There is extensive documentation about the [CodeQL language](https://codeql.github.com/docs/), writing CodeQL using the [CodeQL extension for Visual Studio Code](https://docs.github.com/en/code-security/codeql-for-vs-code/) and using the [CodeQL CLI](https://docs.github.com/en/code-security/codeql-cli).
+There is [extensive documentation](https://codeql.github.com/docs/) on getting started with writing CodeQL using the [CodeQL extension for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/) and the [CodeQL CLI](https://codeql.github.com/docs/codeql-cli/).
 
 ## Contributing
 

WORKSPACE.bazel

@@ -0,0 +1,2 @@
+# please use MODULE.bazel to add dependencies
+# this empty file is required by internal repositories, don't remove it

actions/BUILD.bazel

@@ -1,20 +0,0 @@
-load("//misc/bazel:pkg.bzl", "codeql_pack")
-
-package(default_visibility = ["//visibility:public"])
-
-[
-    codeql_pack(
-        name = "-".join(parts),
-        srcs = [
-            "//actions/extractor",
-        ],
-        pack_prefix = "/".join(parts),
-    )
-    for parts in (
-        [
-            "experimental",
-            "actions",
-        ],
-        ["actions"],
-    )
-]

actions/extractor/BUILD.bazel

@@ -1,10 +0,0 @@
-load("//misc/bazel:pkg.bzl", "codeql_pkg_files", "strip_prefix")
-
-codeql_pkg_files(
-    name = "extractor",
-    srcs = [
-        "codeql-extractor.yml",
-    ] + glob(["tools/**"]),
-    strip_prefix = strip_prefix.from_pkg(),
-    visibility = ["//actions:__pkg__"],
-)

actions/extractor/codeql-extractor.yml

@@ -1,44 +0,0 @@
-name: "actions"
-aliases: []
-display_name: "GitHub Actions"
-version: 0.0.1
-column_kind: "utf16"
-unicode_newlines: true
-build_modes:
-  - none
-file_coverage_languages: []
-github_api_languages: []
-scc_languages: []
-file_types:
-  - name: workflow
-    display_name: GitHub Actions workflow files
-    extensions:
-      - .yml
-      - .yaml
-forwarded_extractor_name: javascript
-options:
-  trap:
-    title: TRAP options
-    description: Options about how the extractor handles TRAP files
-    type: object
-    visibility: 3
-    properties:
-      cache:
-        title: TRAP cache options
-        description: Options about how the extractor handles its TRAP cache
-        type: object
-        properties:
-          dir:
-            title: TRAP cache directory
-            description: The directory of the TRAP cache to use
-            type: string
-          bound:
-            title: TRAP cache bound
-            description: A soft limit (in MB) on the size of the TRAP cache
-            type: string
-            pattern: "[0-9]+"
-          write:
-            title: TRAP cache writeable
-            description: Whether to write to the TRAP cache as well as reading it
-            type: string
-            pattern: "(true|TRUE|false|FALSE)"

actions/extractor/tools/autobuild-impl.ps1

@@ -1,40 +0,0 @@
-if (($null -ne $env:LGTM_INDEX_INCLUDE) -or ($null -ne $env:LGTM_INDEX_EXCLUDE) -or ($null -ne $env:LGTM_INDEX_FILTERS)) {
-    Write-Output 'Path filters set. Passing them through to the JavaScript extractor.' 
-} else {
-    Write-Output 'No path filters set. Using the default filters.'
-    $DefaultPathFilters = @(
-        'exclude:**/*',
-        'include:.github/workflows/**/*.yml',
-        'include:.github/workflows/**/*.yaml',
-        'include:**/action.yml',
-        'include:**/action.yaml'
-    )
-
-    $env:LGTM_INDEX_FILTERS = $DefaultPathFilters -join "`n"
-}
-
-# Find the JavaScript extractor directory via `codeql resolve extractor`.
-$CodeQL = Join-Path $env:CODEQL_DIST 'codeql.exe'
-$env:CODEQL_EXTRACTOR_JAVASCRIPT_ROOT = &$CodeQL resolve extractor --language javascript
-if ($LASTEXITCODE -ne 0) {
-    throw 'Failed to resolve JavaScript extractor.'
-}
-
-Write-Output "Found JavaScript extractor at '${env:CODEQL_EXTRACTOR_JAVASCRIPT_ROOT}'."
-
-# Run the JavaScript autobuilder.
-$JavaScriptAutoBuild = Join-Path $env:CODEQL_EXTRACTOR_JAVASCRIPT_ROOT 'tools\autobuild.cmd'
-Write-Output "Running JavaScript autobuilder at '${JavaScriptAutoBuild}'."
-
-# Copy the values of the Actions extractor environment variables to the JavaScript extractor environment variables.
-$env:CODEQL_EXTRACTOR_JAVASCRIPT_DIAGNOSTIC_DIR = $env:CODEQL_EXTRACTOR_ACTIONS_DIAGNOSTIC_DIR
-$env:CODEQL_EXTRACTOR_JAVASCRIPT_LOG_DIR = $env:CODEQL_EXTRACTOR_ACTIONS_LOG_DIR
-$env:CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR = $env:CODEQL_EXTRACTOR_ACTIONS_SCRATCH_DIR
-$env:CODEQL_EXTRACTOR_JAVASCRIPT_SOURCE_ARCHIVE_DIR = $env:CODEQL_EXTRACTOR_ACTIONS_SOURCE_ARCHIVE_DIR
-$env:CODEQL_EXTRACTOR_JAVASCRIPT_TRAP_DIR = $env:CODEQL_EXTRACTOR_ACTIONS_TRAP_DIR
-$env:CODEQL_EXTRACTOR_JAVASCRIPT_WIP_DATABASE = $env:CODEQL_EXTRACTOR_ACTIONS_WIP_DATABASE
-
-&$JavaScriptAutoBuild
-if ($LASTEXITCODE -ne 0) {
-    throw "JavaScript autobuilder failed."
-}

actions/extractor/tools/autobuild.cmd

@@ -1,3 +0,0 @@
-@echo off
-rem All of the work is done in the PowerShell script
-powershell.exe %~dp0autobuild-impl.ps1

actions/extractor/tools/autobuild.sh

@@ -1,39 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-DEFAULT_PATH_FILTERS=$(cat << END
-exclude:**/*
-include:.github/workflows/**/*.yml
-include:.github/workflows/**/*.yaml
-include:**/action.yml
-include:**/action.yaml
-END
-)
-
-if [ -n "${LGTM_INDEX_INCLUDE:-}" ] || [ -n "${LGTM_INDEX_EXCLUDE:-}" ] || [ -n "${LGTM_INDEX_FILTERS:-}" ] ; then
-    echo "Path filters set. Passing them through to the JavaScript extractor."
-else
-    echo "No path filters set. Using the default filters."
-    LGTM_INDEX_FILTERS="${DEFAULT_PATH_FILTERS}"
-    export LGTM_INDEX_FILTERS
-fi
-
-# Find the JavaScript extractor directory via `codeql resolve extractor`.
-CODEQL_EXTRACTOR_JAVASCRIPT_ROOT="$($CODEQL_DIST/codeql resolve extractor --language javascript)"
-export CODEQL_EXTRACTOR_JAVASCRIPT_ROOT
-
-echo "Found JavaScript extractor at '${CODEQL_EXTRACTOR_JAVASCRIPT_ROOT}'."
-
-# Run the JavaScript autobuilder
-JAVASCRIPT_AUTO_BUILD="${CODEQL_EXTRACTOR_JAVASCRIPT_ROOT}/tools/autobuild.sh"
-echo "Running JavaScript autobuilder at '${JAVASCRIPT_AUTO_BUILD}'."
-
-# Copy the values of the Actions extractor environment variables to the JavaScript extractor environment variables.
-env CODEQL_EXTRACTOR_JAVASCRIPT_DIAGNOSTIC_DIR="${CODEQL_EXTRACTOR_ACTIONS_DIAGNOSTIC_DIR}" \
-    CODEQL_EXTRACTOR_JAVASCRIPT_LOG_DIR="${CODEQL_EXTRACTOR_ACTIONS_LOG_DIR}" \
-    CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR="${CODEQL_EXTRACTOR_ACTIONS_SCRATCH_DIR}" \
-    CODEQL_EXTRACTOR_JAVASCRIPT_SOURCE_ARCHIVE_DIR="${CODEQL_EXTRACTOR_ACTIONS_SOURCE_ARCHIVE_DIR}" \
-    CODEQL_EXTRACTOR_JAVASCRIPT_TRAP_DIR="${CODEQL_EXTRACTOR_ACTIONS_TRAP_DIR}" \
-    CODEQL_EXTRACTOR_JAVASCRIPT_WIP_DATABASE="${CODEQL_EXTRACTOR_ACTIONS_WIP_DATABASE}" \
-    ${JAVASCRIPT_AUTO_BUILD}

actions/ql/lib/actions.qll

@@ -1 +0,0 @@
-predicate placeholder(int x) { x = 0 }

actions/ql/lib/qlpack.yml

@@ -1,12 +0,0 @@
-name: codeql/actions-all
-version: 0.0.1-dev
-library: true
-warnOnImplicitThis: true
-dependencies:
-  codeql/util: ${workspace}
-  codeql/yaml: ${workspace}
-  codeql/controlflow: ${workspace}
-  codeql/dataflow: ${workspace}
-  codeql/javascript-all: ${workspace}
-extractor: actions
-groups: actions

actions/ql/src/Placeholder.ql

@@ -1,16 +0,0 @@
-/**
- * @name Placeholder Query
- * @description Placeholder
- * @kind problem
- * @problem.severity warning
- * @security-severity 9.3
- * @precision high
- * @id actions/placeholder
- * @tags actions security
- */
-
-import actions
-import javascript
-
-from File f
-select f, "Analyzed a file."

actions/ql/src/qlpack.yml

@@ -1,8 +0,0 @@
-name: codeql/actions-queries
-version: 0.0.1-dev
-library: false
-groups: [actions, queries]
-extractor: actions
-dependencies:
-  codeql/actions-all: ${workspace}
-warnOnImplicitThis: true

actions/ql/test/library-tests/.github/workflows/shell.yml

@@ -1,23 +0,0 @@
-on: push
-
-jobs:
-  job1:
-    runs-on: ubuntu-latest
-    steps:
-      - shell: pwsh
-        run: Write-Output "foo"
-  job2:
-    runs-on: ubuntu-latest
-    steps:
-      - run: echo "foo"
-
-  job3:
-    runs-on: windows-latest
-    steps:
-      - shell: bash
-        run: echo "foo"
-  job4:
-    runs-on: windows-latest
-    steps:
-      - run: Write-Output "foo"
-

actions/ql/test/library-tests/Placeholder.expected

@@ -1 +0,0 @@
-| 1 |

actions/ql/test/library-tests/Placeholder.ql

@@ -1 +0,0 @@
-select 1

actions/ql/test/qlpack.yml

@@ -1,8 +0,0 @@
-name: codeql/actions-tests
-groups: [codeql, test]
-dependencies:
-  codeql/actions-all: ${workspace}
-  codeql/actions-queries: ${workspace}
-extractor: actions
-tests: .
-warnOnImplicitThis: true

actions/ql/test/query-tests/Placeholder/.github/workflows/shell.yml

@@ -1,23 +0,0 @@
-on: push
-
-jobs:
-  job1:
-    runs-on: ubuntu-latest
-    steps:
-      - shell: pwsh
-        run: Write-Output "foo"
-  job2:
-    runs-on: ubuntu-latest
-    steps:
-      - run: echo "foo"
-
-  job3:
-    runs-on: windows-latest
-    steps:
-      - shell: bash
-        run: echo "foo"
-  job4:
-    runs-on: windows-latest
-    steps:
-      - run: Write-Output "foo"
-

actions/ql/test/query-tests/Placeholder/Placeholder.expected

@@ -1 +0,0 @@
-| .github/workflows/shell.yml:0:0:0:0 | .github/workflows/shell.yml | Analyzed a file. |

actions/ql/test/query-tests/Placeholder/Placeholder.qlref

@@ -1 +0,0 @@
-Placeholder.ql

codeql-workspace.yml

@@ -6,16 +6,19 @@ provide:
   - "*/ql/consistency-queries/qlpack.yml"
   - "*/ql/automodel/src/qlpack.yml"
   - "*/ql/automodel/test/qlpack.yml"
-  - "*/extractor-pack/codeql-extractor.yml"
   - "python/extractor/qlpack.yml"
   - "shared/**/qlpack.yml"
   - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
   - "go/ql/config/legacy-support/qlpack.yml"
+  - "go/build/codeql-extractor-go/codeql-extractor.yml"
   - "csharp/ql/campaigns/Solorigate/lib/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/src/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/test/qlpack.yml"
   - "misc/legacy-support/*/qlpack.yml"
   - "misc/suite-helpers/qlpack.yml"
+  - "ruby/extractor-pack/codeql-extractor.yml"
+  - "swift/extractor-pack/codeql-extractor.yml"
+  - "ql/extractor-pack/codeql-extractor.yml"
   - ".github/codeql/extensions/**/codeql-pack.yml"
 
 versionPolicies:

config/dbscheme-fragments.json

@@ -28,7 +28,6 @@
     "/*- Yaml dbscheme -*/",
     "/*- Blame dbscheme -*/",
     "/*- JSON dbscheme -*/",
-    "/*- Python dbscheme -*/",
-    "/*- Empty location -*/"
+    "/*- Python dbscheme -*/"
   ]
 }

config/identical-files.json

@@ -57,6 +57,14 @@
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll"
   ],
+  "Model as Data Generation Java/C# - CaptureModels": [
+    "java/ql/src/utils/modelgenerator/internal/CaptureModels.qll",
+    "csharp/ql/src/utils/modelgenerator/internal/CaptureModels.qll"
+  ],
+  "Model as Data Generation Java/C# - CaptureModelsPrinting": [
+    "java/ql/src/utils/modelgenerator/internal/CaptureModelsPrinting.qll",
+    "csharp/ql/src/utils/modelgenerator/internal/CaptureModelsPrinting.qll"
+  ],
   "Sign Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/Sign.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/Sign.qll"
@@ -177,6 +185,11 @@
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/AliasAnalysisImports.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/AliasAnalysisImports.qll"
   ],
+  "C++ IR ValueNumberingImports": [
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/gvn/internal/ValueNumberingImports.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/gvn/internal/ValueNumberingImports.qll",
+    "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/gvn/internal/ValueNumberingImports.qll"
+  ],
   "IR SSA SSAConstruction": [
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/SSAConstruction.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/SSAConstruction.qll"

cpp/autobuilder/.gitignore

@@ -0,0 +1,13 @@
+obj/
+TestResults/
+*.manifest
+*.pdb
+*.suo
+*.mdb
+*.vsmdi
+csharp.log
+**/bin/Debug
+**/bin/Release
+*.tlog
+.vs
+*.user

cpp/autobuilder/README.md

@@ -1 +0,0 @@
-The Windows autobuilder that used to live in this directory moved to `csharp/autobuilder/Semmle.Autobuild.Cpp`.

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -7,7 +7,6 @@ using System.Linq;
 using Microsoft.Build.Construction;
 using System.Xml;
 using System.IO;
-using Semmle.Util.Logging;
 
 namespace Semmle.Autobuild.Cpp.Tests
 {
@@ -190,7 +189,7 @@ namespace Semmle.Autobuild.Cpp.Tests
                 throw new ArgumentException($"Missing CreateDirectory, {path}");
         }
 
-        public void DownloadFile(string address, string fileName, ILogger logger)
+        public void DownloadFile(string address, string fileName)
         {
             if (!DownloadFiles.Contains((address, fileName)))
                 throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");
@@ -201,9 +200,9 @@ namespace Semmle.Autobuild.Cpp.Tests
 
     internal class TestDiagnosticWriter : IDiagnosticsWriter
     {
-        public IList<Semmle.Util.DiagnosticMessage> Diagnostics { get; } = new List<Semmle.Util.DiagnosticMessage>();
+        public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
 
-        public void AddEntry(Semmle.Util.DiagnosticMessage message) => this.Diagnostics.Add(message);
+        public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
 
         public void Dispose() { }
     }

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj

@@ -0,0 +1,26 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
+    <PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
+    <PackageReference Include="xunit" Version="2.6.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.4">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Semmle.Autobuild.Cpp\Semmle.Autobuild.Cpp.csproj" />
+    <ProjectReference Include="..\..\..\csharp\autobuilder\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
+  </ItemGroup>
+</Project>

cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs

@@ -0,0 +1,35 @@
+using System;
+
+using Semmle.Autobuild.Shared;
+using Semmle.Util;
+
+namespace Semmle.Autobuild.Cpp
+{
+    class Program
+    {
+        static int Main()
+        {
+
+            try
+            {
+                var actions = SystemBuildActions.Instance;
+                var options = new CppAutobuildOptions(actions);
+                try
+                {
+                    Console.WriteLine("CodeQL C++ autobuilder");
+                    using var builder = new CppAutobuilder(actions, options);
+                    return builder.AttemptBuild();
+                }
+                catch (InvalidEnvironmentException ex)
+                {
+                    Console.WriteLine("The environment is invalid: {0}", ex.Message);
+                }
+            }
+            catch (ArgumentOutOfRangeException ex)
+            {
+                Console.WriteLine("The value \"{0}\" for parameter \"{1}\" is invalid", ex.ActualValue, ex.ParamName);
+            }
+            return 1;
+        }
+    }
+}

cpp/autobuilder/Semmle.Autobuild.Cpp/Properties/AssemblyInfo.cs

@@ -0,0 +1,32 @@
+using System.Reflection;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("Semmle.Autobuild.Cpp")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("GitHub")]
+[assembly: AssemblyProduct("CodeQL autobuilder for C++")]
+[assembly: AssemblyCopyright("Copyright © GitHub 2020")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components.  If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <AssemblyName>Semmle.Autobuild.Cpp</AssemblyName>
+    <RootNamespace>Semmle.Autobuild.Cpp</RootNamespace>
+    <ApplicationIcon />
+    <OutputType>Exe</OutputType>
+    <StartupObject />
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Folder Include="Properties\" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Build" Version="17.8.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\csharp\extractor\Semmle.Util\Semmle.Util.csproj" />
+    <ProjectReference Include="..\..\..\csharp\autobuilder\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
+  </ItemGroup>
+
+</Project>

cpp/downgrades/02a123a1a681f98cf502f189a2a79b0dfb398e59/exprs.ql

@@ -1,17 +0,0 @@
-class Expr extends @expr {
-  string toString() { none() }
-}
-
-class Location extends @location_expr {
-  string toString() { none() }
-}
-
-predicate isExprWithNewBuiltin(Expr expr) {
-  exists(int kind | exprs(expr, kind, _) | 385 <= kind and kind <= 388)
-}
-
-from Expr expr, int kind, int kind_new, Location location
-where
-  exprs(expr, kind, location) and
-  if isExprWithNewBuiltin(expr) then kind_new = 1 else kind_new = kind
-select expr, kind_new, location

cpp/downgrades/02a123a1a681f98cf502f189a2a79b0dfb398e59/sizeof_bind.ql

@@ -1,14 +0,0 @@
-class Expr extends @expr {
-  string toString() { none() }
-}
-
-class Type extends @type {
-  string toString() { none() }
-}
-
-from Expr expr, Type type, int kind
-where
-  sizeof_bind(expr, type) and
-  exprs(expr, kind, _) and
-  (kind = 93 or kind = 94)
-select expr, type

cpp/downgrades/02a123a1a681f98cf502f189a2a79b0dfb398e59/upgrade.properties

@@ -1,4 +0,0 @@
-description: Add new builtin operations
-compatibility: partial
-exprs.rel: run exprs.qlo
-sizeof_bind.rel: run sizeof_bind.qlo

cpp/downgrades/0fea0ee7026c7c3f7d6faef4df4bf67847b67d71/downgrades.ql

@@ -1,32 +0,0 @@
-/*
- * Approach: replace conversion expressions of kind 389 (= @c11_generic) by
- * conversion expressions of kind 12 (= @parexpr), i.e., a `ParenthesisExpr`,
- * and drop the relation which its child expressions, which are just syntactic
- * sugar. Parenthesis expressions are equally benign as C11 _Generic expressions,
- * and behave similarly in the context of the IR.
- */
-
-class Expr extends @expr {
-  string toString() { none() }
-}
-
-class Location extends @location {
-  string toString() { none() }
-}
-
-class ExprParent extends @exprparent {
-  string toString() { none() }
-}
-
-query predicate new_exprs(Expr expr, int new_kind, Location loc) {
-  exists(int kind | exprs(expr, kind, loc) | if kind = 389 then new_kind = 12 else new_kind = kind)
-}
-
-query predicate new_exprparents(Expr expr, int index, ExprParent expr_parent) {
-  exprparents(expr, index, expr_parent) and
-  (
-    not expr_parent instanceof @expr
-    or
-    exists(int kind | exprs(expr_parent.(Expr), kind, _) | kind != 389)
-  )
-}

cpp/downgrades/0fea0ee7026c7c3f7d6faef4df4bf67847b67d71/upgrade.properties

@@ -1,4 +0,0 @@
-description: Expose C11 _Generics
-compatibility: partial
-exprs.rel: run downgrades.ql new_exprs
-exprparents.rel: run downgrades.ql new_exprparents

cpp/downgrades/19887dbd33327fb07d54251786e0cb2578539775/upgrade.properties

@@ -1,4 +1,4 @@
 description: Revert support for repeated initializers, which are allowed in C with designated initializers.
 compatibility: full
-aggregate_field_init.rel: reorder aggregate_field_init.rel (@aggregateliteral aggregate, @expr initializer, @membervariable field, int position) aggregate initializer field
-aggregate_array_init.rel: reorder aggregate_array_init.rel (@aggregateliteral aggregate, @expr initializer, int element_index, int position) aggregate initializer element_index
+aggregate_field_init.rel: reorder aggregate_field_init.rel (int aggregate, int initializer, int field, int position) aggregate initializer field
+aggregate_array_init.rel: reorder aggregate_array_init.rel (int aggregate, int initializer, int element_index, int position) aggregate initializer element_index

cpp/downgrades/25e365d1e8147df0f759b604f96eb4bffea48271/upgrade.properties

@@ -1,4 +0,0 @@
-description: Revert support for using-enum declarations.
-compatibility: partial
-usings.rel: run usings.qlo
-using_container.rel: run using_container.qlo

cpp/downgrades/25e365d1e8147df0f759b604f96eb4bffea48271/using_container.ql

@@ -1,14 +0,0 @@
-class UsingEntry extends @using {
-  string toString() { none() }
-}
-
-class Element extends @element {
-  string toString() { none() }
-}
-
-from UsingEntry u, Element parent, int kind
-where
-  usings(u, _, _, kind) and
-  using_container(parent, u) and
-  kind != 3
-select parent, u

cpp/downgrades/25e365d1e8147df0f759b604f96eb4bffea48271/usings.ql

@@ -1,17 +0,0 @@
-class UsingEntry extends @using {
-  string toString() { none() }
-}
-
-class Element extends @element {
-  string toString() { none() }
-}
-
-class Location extends @location_default {
-  string toString() { none() }
-}
-
-from UsingEntry u, Element target, Location loc, int kind
-where
-  usings(u, target, loc, kind) and
-  kind != 3
-select u, target, loc

cpp/downgrades/3d35dd6b50edfc540c14c6757e0c7b3c5b7b04dd/exprs.ql

@@ -1,17 +0,0 @@
-class Expr extends @expr {
-  string toString() { none() }
-}
-
-class Location extends @location_expr {
-  string toString() { none() }
-}
-
-predicate isExprWithNewBuiltin(Expr expr) {
-  exists(int kind | exprs(expr, kind, _) | 364 <= kind and kind <= 384)
-}
-
-from Expr expr, int kind, int kind_new, Location location
-where
-  exprs(expr, kind, location) and
-  if isExprWithNewBuiltin(expr) then kind_new = 1 else kind_new = kind
-select expr, kind_new, location

cpp/downgrades/3d35dd6b50edfc540c14c6757e0c7b3c5b7b04dd/upgrade.properties

@@ -1,3 +0,0 @@
-description: Add new builtin operations
-compatibility: partial
-exprs.rel: run exprs.qlo

cpp/downgrades/68930f3b81bbe3fdbb91c850deca1fec8072d62a/upgrade.properties

@@ -1,3 +0,0 @@
-description: description: Support explicit(bool) specifiers
-compatibility: full
-explicit_specifier_exprs.rel: delete

cpp/downgrades/6f5d51e89e762fe4609fd4ac8ee3afb04221e873/exprs.ql

@@ -1,15 +0,0 @@
-class Expr extends @expr {
-  string toString() { none() }
-}
-
-class Location extends @location_expr {
-  string toString() { none() }
-}
-
-predicate isExprRequires(Expr expr) { exists(int kind | exprs(expr, kind, _) | kind = 390) }
-
-from Expr expr, int kind, int kind_new, Location location
-where
-  exprs(expr, kind, location) and
-  if isExprRequires(expr) then kind_new = 1 else kind_new = kind
-select expr, kind_new, location