DO NOT MERGE: Add meta queries to security extended for QA

This reverts commit 76f6adde5121d6d0740b419e08c083d48e887d2f.

.bazelrc

@@ -1,47 +1,9 @@
 common --enable_platform_specific_config
-# because we use --override_module with `%workspace%`, the lock file is not stable
-common --lockfile_mode=off
-
-# Build release binaries by default, can be overwritten to in local.bazelrc and set to `fastbuild` or `dbg`
-build --compilation_mode opt
-
-# when building from this repository in isolation, the internal repository will not be found at ..
-# where `MODULE.bazel` looks for it. The following will get us past the module loading phase, so
-# that we can build things that do not rely on that
-common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
 
 build --repo_env=CC=clang --repo_env=CXX=clang++
-# Disable Android SDK auto-detection (we don't use it, and rules_android has Bazel 9 compatibility issues)
-build --repo_env=ANDROID_HOME=
 
-# print test output, like sembuild does.
-# Set to `errors` if this is too verbose.
-test --test_output all
-# we use transitions that break builds of `...`, so for `test` to work with that we need the following
-test --build_tests_only
-
-# this requires developer mode, but is required to have pack installer functioning
-startup --windows_enable_symlinks
-common --enable_runfiles
-
-# with the above, we can avoid building python zips which is the default on windows as that's expensive
-build --nobuild_python_zip
-
-common --registry=file:///%workspace%/misc/bazel/registry
-common --registry=https://bcr.bazel.build
-
-common --@rules_dotnet//dotnet/settings:strict_deps=false
-
-# we only configure a nightly toolchain
-common --@rules_rust//rust/toolchain/channel=nightly
-
-# Reduce this eventually to empty, once we've fixed all our usages of java, and https://github.com/bazel-contrib/rules_go/issues/4193 is fixed
-common --incompatible_autoload_externally="+@rules_cc,+@rules_java,+@rules_shell"
-
-build --java_language_version=17
-build --tool_java_language_version=17
-build --tool_java_runtime_version=remotejdk_17
-build --java_runtime_version=remotejdk_17
-build --@rules_python//python/config_settings:python_version=3.12
+build:linux --cxxopt=-std=c++20
+build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
+build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
 
 try-import %workspace%/local.bazelrc

.bazelrc.internal

@@ -1,12 +0,0 @@
-# this file should contain bazel settings required to build things from `semmle-code`
-
-common --registry=file:///%workspace%/ql/misc/bazel/registry
-common --registry=https://bcr.bazel.build
-
-# See bazelbuild/rules_dotnet#413: strict_deps in C# also appliy to 3rd-party deps, and when we pull
-# in (for example) the xunit package, there's no code in this at all, it just depends transitively on
-# its implementation packages without providing any code itself.
-# We either can depend on internal implementation details, or turn of strict deps.
-common --@rules_dotnet//dotnet/settings:strict_deps=false
-
-build --@rules_python//python/config_settings:python_version=3.12

.bazelversion

@@ -1 +1 @@
-9.0.0
+6.3.1

.clang-format

@@ -1 +0,0 @@
-DisableFormat: true

.devcontainer/Dockerfile.codespaces

@@ -1,7 +0,0 @@
-FROM mcr.microsoft.com/devcontainers/base:ubuntu-24.04
-
-USER root
-# Install needed packages according to https://codeql.github.com/docs/codeql-overview/system-requirements/
-# most come from the base image, but we need to install some additional ones
-RUN DEBIAN_FRONTEND=noninteractive apt update && apt install -y sudo man-db python3.12 npm unminimize
-RUN yes | unminimize

.devcontainer/devcontainer.json

@@ -7,10 +7,6 @@
     "ms-vscode.test-adapter-converter",
     "slevesque.vscode-zipexplorer"
   ],
-  "build": {
-    // Path is relative to the devcontainer.json file.
-    "dockerfile": "Dockerfile.codespaces"
-  },
   "settings": {
     "files.watcherExclude": {
       "**/target/**": true

.devcontainer/swift/Dockerfile

@@ -0,0 +1,9 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.236.0/containers/cpp/.devcontainer/base.Dockerfile
+
+# [Choice] Debian / Ubuntu version (use Debian 11, Ubuntu 18.04/22.04 on local arm64/Apple Silicon): debian-11, debian-10, ubuntu-22.04, ubuntu-20.04, ubuntu-18.04
+FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-ubuntu-22.04
+
+USER root
+ADD root.sh /tmp/root.sh
+ADD update-codeql.sh /usr/local/bin/update-codeql
+RUN bash /tmp/root.sh && rm /tmp/root.sh

.devcontainer/swift/devcontainer.json

@@ -0,0 +1,25 @@
+{
+	"extensions": [
+		"github.vscode-codeql",
+		"hbenl.vscode-test-explorer",
+		"ms-vscode.test-adapter-converter",
+		"slevesque.vscode-zipexplorer",
+		"ms-vscode.cpptools"
+	],
+	"settings": {
+		"files.watcherExclude": {
+			"**/target/**": true
+		},
+		"codeQL.runningQueries.memory": 2048
+	},
+	"build": {
+		"dockerfile": "Dockerfile",
+	},
+	"runArgs": [
+		"--cap-add=SYS_PTRACE",
+		"--security-opt",
+		"seccomp=unconfined"
+	],
+	"remoteUser": "vscode",
+	"onCreateCommand": ".devcontainer/swift/user.sh"
+}

.devcontainer/swift/root.sh

@@ -0,0 +1,22 @@
+set -xe
+
+BAZELISK_VERSION=v1.12.0
+BAZELISK_DOWNLOAD_SHA=6b0bcb2ea15bca16fffabe6fda75803440375354c085480fe361d2cbf32501db
+
+apt-get update
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y install --no-install-recommends \
+    zlib1g-dev \
+    uuid-dev \
+    python3-distutils \
+    python3-pip \
+    bash-completion
+
+# Install Bazel
+curl -fSsL -o /usr/local/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-amd64
+echo "${BAZELISK_DOWNLOAD_SHA} */usr/local/bin/bazelisk" | sha256sum --check -
+chmod 0755 /usr/local/bin/bazelisk
+ln -s bazelisk /usr/local/bin/bazel
+
+# install latest codeql
+update-codeql

.devcontainer/swift/update-codeql.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+
+URL=https://github.com/github/codeql-cli-binaries/releases
+LATEST_VERSION=$(curl -L -s -H 'Accept: application/json' $URL/latest | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/')
+CURRENT_VERSION=v$(codeql version 2>/dev/null | sed -ne 's/.*release \([0-9.]*\)\./\1/p')
+if [[ $CURRENT_VERSION != $LATEST_VERSION ]]; then
+  if [[ $UID != 0 ]]; then
+    echo "update required, please run this script with sudo:"
+    echo "  sudo $0"
+    exit 1
+  fi
+  ZIP=$(mktemp codeql.XXXX.zip)
+  curl -fSqL -o $ZIP $URL/download/$LATEST_VERSION/codeql-linux64.zip
+  unzip -q $ZIP -d /opt
+  rm $ZIP
+  ln -sf /opt/codeql/codeql /usr/local/bin/codeql
+  echo installed version $LATEST_VERSION
+else
+  echo current version $CURRENT_VERSION is up-to-date
+fi

.devcontainer/swift/user.sh

@@ -0,0 +1,13 @@
+set -xe
+
+# add the workspace to the codeql search path
+mkdir -p /home/vscode/.config/codeql
+echo "--search-path /workspaces/codeql" > /home/vscode/.config/codeql/config
+
+# create a swift extractor pack with the current state
+cd /workspaces/codeql
+bazel run swift/create-extractor-pack
+
+#install and set up pre-commit
+python3 -m pip install pre-commit --no-warn-script-location
+$HOME/.local/bin/pre-commit install

.gitattributes

@@ -50,38 +50,24 @@
 *.dll -text
 *.pdb -text
 
-/java/ql/test/stubs/**/*.java linguist-generated=true
-/java/ql/test/experimental/stubs/**/*.java linguist-generated=true
-/java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text
+java/ql/test/stubs/**/*.java linguist-generated=true
+java/ql/test/experimental/stubs/**/*.java linguist-generated=true
 
 # Force git not to modify line endings for go or html files under the go/ql directory
-/go/ql/**/*.go -text
-/go/ql/**/*.html -text
+go/ql/**/*.go -text
+go/ql/**/*.html -text
 # Force git not to modify line endings for go dbschemes
-/go/*.dbscheme -text
+go/*.dbscheme -text
 # Preserve unusual line ending from codeql-go merge
-/go/extractor/opencsv/CSVReader.java -text
+go/extractor/opencsv/CSVReader.java -text
 
 # For some languages, upgrade script testing references really old dbscheme
 # files from legacy upgrades that have CRLF line endings. Since upgrade
 # resolution relies on object hashes, we must suppress line ending conversion
 # for those testing dbscheme files.
-/*/ql/lib/upgrades/initial/*.dbscheme -text
+*/ql/lib/upgrades/initial/*.dbscheme -text
 
-# Auto-generated modeling for Python
-/python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
-
-# auto-generated bazel lock file
-/ruby/extractor/cargo-bazel-lock.json linguist-generated=true
-/ruby/extractor/cargo-bazel-lock.json -merge
-
-# auto-generated files for the C# build
-/csharp/paket.lock linguist-generated=true
-# needs eol=crlf, as `paket` touches this file and saves it as crlf
-/csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf
-/csharp/paket.main.bzl linguist-generated=true
-/csharp/paket.main_extension.bzl linguist-generated=true
-
-# swift prebuilt resources
-/swift/third_party/resources/*.zip filter=lfs diff=lfs merge=lfs -text
-/swift/third_party/resources/*.tar.zst filter=lfs diff=lfs merge=lfs -text
+# Generated test files - these are synced from the standard JavaScript libraries using
+# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
+javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
+javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge

.github/codeql/codeql-config.yml

@@ -4,13 +4,8 @@ queries:
   - uses: security-and-quality
 
 paths-ignore:
-  - '/actions/ql/test'
   - '/cpp/'
   - '/java/'
   - '/python/'
   - '/javascript/ql/test'
-  - '/javascript/ql/integration-tests'
   - '/javascript/extractor/tests'
-  - '/javascript/extractor/parser-tests'
-  - '/javascript/ql/src/'
-  - '/rust/ql'

.github/dependabot.yml

@@ -40,10 +40,3 @@ updates:
       - dependency-name: "*"
     reviewers:
       - "github/codeql-go"
-
-  - package-ecosystem: bazel
-    directory: "/"
-    schedule:
-      interval: weekly
-    exclude-paths:
-      - "misc/bazel/registry/**"

.github/instructions/expected-files.instructions.md

@@ -1,4 +0,0 @@
----
-applyTo: "**/*.expected"
----
-Ignore these files. These are generated to be correct by running the tests, and should not be edited manually.

.github/instructions/ql-files.instructions.md

@@ -1,6 +0,0 @@
----
-applyTo: "**/*.ql,**/*.qll"
----
-When reviewing these QL files, do not make any comment on the code itself. You don't understand the QL programming language well enough yet. You can only make comments about the following aspects:
-* comments (lines starting with `//` or block comments enclosed in `/* ... */`): you can suggest improvements to the clarity of comments, or point out spelling mistakes
-* typos in identifiers

.github/labeler.yml

@@ -15,12 +15,12 @@ Java:
   - change-notes/**/*java.*
 
 JS:
-  - any: [ 'javascript/**/*' ]
+  - any: [ 'javascript/**/*', '!javascript/ql/experimental/adaptivethreatmodeling/**/*' ]
   - change-notes/**/*javascript*
 
 Kotlin:
   - java/kotlin-extractor/**/*
-  - java/ql/test-kotlin*/**/*
+  - java/ql/test/kotlin/**/*
 
 Python:
   - python/**/*
@@ -30,18 +30,10 @@ Ruby:
   - ruby/**/*
   - change-notes/**/*ruby*
 
-Rust:
-  - rust/**/*
-  - change-notes/**/*rust*
-
 Swift:
   - swift/**/*
   - change-notes/**/*swift*
 
-Actions:
-  - actions/**/*
-  - change-notes/**/*actions*
-
 documentation:
   - "**/*.qhelp"
   - "**/*.md"
@@ -54,3 +46,6 @@ documentation:
 # Since these are all shared files that need to be synced, just pick _one_ copy of each.
 "DataFlow Library":
   - "shared/dataflow/**/*"
+
+"ATM":
+  - javascript/ql/experimental/adaptivethreatmodeling/**/*

.github/workflows/buildifier.yml

@@ -1,28 +0,0 @@
-name: Check bazel formatting
-
-on:
-  pull_request:
-    paths:
-      - "**.bazel"
-      - "**.bzl"
-    branches:
-      - main
-      - "rc/*"
-
-permissions:
-  contents: read
-
-jobs:
-  check:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v5
-      - name: Check bazel formatting
-        uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
-        with:
-          extra_args: >
-            buildifier --all-files 2>&1 ||
-            (
-              echo -e "In order to format all bazel files, please run:\n  bazel run //misc/bazel/buildifier"; exit 1
-            )

.github/workflows/check-change-note.yml

@@ -1,8 +1,5 @@
 name: Check change note
 
-permissions:
-  pull-requests: read
-
 on:
   pull_request_target:
     types: [labeled, unlabeled, opened, synchronize, reopened, ready_for_review]
@@ -20,7 +17,7 @@ on:
 
 jobs:
   check-change-note:
-    env:
+    env: 
       REPO: ${{ github.repository }}
       PULL_REQUEST_NUMBER: ${{ github.event.number }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -33,7 +30,7 @@ jobs:
           !contains(github.event.pull_request.labels.*.name, 'no-change-note-required')
         run: |
           change_note_files=$(gh api "repos/$REPO/pulls/$PULL_REQUEST_NUMBER/files" --paginate --jq '.[].filename | select(test("/change-notes/.*[.]md$"))')
-
+          
           if [ -z "$change_note_files" ]; then
             echo "No change note found. Either add one, or add the 'no-change-note-required' label."
             exit 1

.github/workflows/check-implicit-this.yml

@@ -9,14 +9,11 @@ on:
       - main
       - "rc/*"
 
-permissions:
-  contents: read
-
 jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
       - name: Check that implicit this warnings is enabled for all packs
         shell: bash
         run: |

.github/workflows/check-overlay-annotations.yml

@@ -1,23 +0,0 @@
-name: Check overlay annotations
-
-on:
-  push:
-    branches:
-      - main
-      - 'rc/*'
-  pull_request:
-    branches:
-      - main
-      - 'rc/*'
-
-permissions:
-  contents: read
-
-jobs:
-  sync:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v5
-      - name: Check overlay annotations
-        run: python config/add-overlay-annotations.py --check java
-

.github/workflows/check-qldoc.yml

@@ -10,15 +10,12 @@ on:
       - main
       - "rc/*"
 
-permissions:
-  contents: read
-
 jobs:
   qldoc:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 2
 
@@ -30,8 +27,7 @@ jobs:
         run: |
           EXIT_CODE=0
           # TODO: remove the shared exception from the regex when coverage of qlpacks without dbschemes is supported
-          # TODO: remove the actions exception once https://github.com/github/codeql-team/issues/3656 is fixed
-          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!(shared|actions))[a-z]*/ql/lib' || true; } | sort -u)"
+          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!(shared))[a-z]*/ql/lib' || true; } | sort -u)"
           for pack_dir in ${changed_lib_packs}; do
             lang="${pack_dir%/ql/lib}"
             codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"

.github/workflows/check-query-ids.yml

@@ -11,14 +11,11 @@ on:
       - "rc/*"
   workflow_dispatch:
 
-permissions:
-  contents: read
-
 jobs:
   check:
     name: Check query IDs
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
       - name: Check for duplicate query IDs
         run: python3 misc/scripts/check-query-ids.py

.github/workflows/close-stale.yml

@@ -5,9 +5,6 @@ on:
   schedule:
     - cron: "30 1 * * *"
 
-permissions:
-  issues: write
-
 jobs:
   stale:
     if: github.repository == 'github/codeql'
@@ -15,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/stale@v9
+    - uses: actions/stale@v8
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Comment or remove the `Stale` label in order to avoid having this issue closed in 7 days.'

.github/workflows/codeql-analysis.yml

@@ -18,10 +18,6 @@ on:
 
 jobs:
   CodeQL-Build:
-    strategy:
-      fail-fast: false
-      matrix:
-        language: ['actions', 'csharp']
 
     runs-on: ubuntu-latest
 
@@ -32,18 +28,19 @@ jobs:
 
     steps:
     - name: Setup dotnet
-      uses: actions/setup-dotnet@v4
+      uses: actions/setup-dotnet@v3
       with:
-        dotnet-version: 10.0.100
+        dotnet-version: 7.0.102
 
     - name: Checkout repository
-      uses: actions/checkout@v5
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@main
+      # Override language selection by uncommenting this and choosing your languages
       with:
-        languages: ${{ matrix.language }}
+        languages: csharp
         config-file: ./.github/codeql/codeql-config.yml
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
@@ -59,9 +56,7 @@ jobs:
     #    uses a compiled language
 
     - run: |
-       cd csharp
-       dotnet tool restore
-       dotnet build .
+       dotnet build csharp
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@main

.github/workflows/compile-queries.yml

@@ -0,0 +1,37 @@
+name: "Compile all queries using the latest stable CodeQL CLI"
+
+on:
+  push:
+    branches:  # makes sure the cache gets populated - running on the branches people tend to merge into.
+      - main
+      - "rc/*"
+      - "codeql-cli-*"
+  pull_request:
+
+jobs:
+  compile-queries:
+    runs-on: ubuntu-latest-xl
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup CodeQL
+        uses: ./.github/actions/fetch-codeql
+        with:
+          channel: 'release'
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: all-queries
+      - name: check formatting
+        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 -n 3000 -P 10 codeql query format -q --check-only
+      - name: compile queries - check-only
+        # run with --check-only if running in a PR (github.sha != main)
+        if : ${{ github.event_name == 'pull_request' }}
+        shell: bash
+        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" --compilation-cache-size=500
+      - name: compile queries - full
+        # do full compile if running on main - this populates the cache
+        if : ${{ github.event_name != 'pull_request' }}
+        shell: bash
+        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" --compilation-cache-size=500

.github/workflows/cpp-swift-analysis.yml

@@ -1,53 +0,0 @@
-name: "Code scanning - C++"
-
-on:
-  push:
-    branches:
-      - main
-      - 'rc/*'
-  pull_request:
-    branches:
-      - main
-      - 'rc/*'
-    paths:
-      - 'swift/**'
-      - '.github/codeql/**'
-      - '.github/workflows/cpp-swift-analysis.yml'
-  schedule:
-    - cron: '0 9 * * 1'
-
-jobs:
-  CodeQL-Build:
-
-    runs-on: ubuntu-24.04
-
-    permissions:
-      contents: read
-      security-events: write
-      pull-requests: read
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v5
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@main
-      # Override language selection by uncommenting this and choosing your languages
-      with:
-        languages: cpp
-        config-file: ./.github/codeql/codeql-config.yml
-
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y uuid-dev
-
-    - name: "Build Swift extractor using Bazel"
-      run: |
-        bazel clean --expunge
-        bazel run //swift:install --nouse_action_cache --noremote_accept_cached --noremote_upload_local_results --spawn_strategy=local
-        bazel shutdown
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@main

.github/workflows/csharp-qltest.yml

@@ -5,10 +5,8 @@ on:
     paths:
       - "csharp/**"
       - "shared/**"
-      - "misc/bazel/**"
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
-      - "MODULE.bazel"
     branches:
       - main
       - "rc/*"
@@ -16,11 +14,9 @@ on:
     paths:
       - "csharp/**"
       - "shared/**"
-      - "misc/bazel/**"
       - .github/workflows/csharp-qltest.yml
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml
-      - "MODULE.bazel"
     branches:
       - main
       - "rc/*"
@@ -29,33 +25,67 @@ defaults:
   run:
     working-directory: csharp
 
-permissions:
-  contents: read
-
 jobs:
+  qlupgrade:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/fetch-codeql
+      - name: Check DB upgrade scripts
+        run: |
+          echo >empty.trap
+          codeql dataset import -S ql/lib/upgrades/initial/semmlecode.csharp.dbscheme testdb empty.trap
+          codeql dataset upgrade testdb --additional-packs ql/lib
+          diff -q testdb/semmlecode.csharp.dbscheme ql/lib/semmlecode.csharp.dbscheme
+      - name: Check DB downgrade scripts
+        run: |
+          echo >empty.trap
+          rm -rf testdb; codeql dataset import -S ql/lib/semmlecode.csharp.dbscheme testdb empty.trap
+          codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
+           --dbscheme=ql/lib/semmlecode.csharp.dbscheme --target-dbscheme=downgrades/initial/semmlecode.csharp.dbscheme |
+           xargs codeql execute upgrades testdb
+          diff -q testdb/semmlecode.csharp.dbscheme downgrades/initial/semmlecode.csharp.dbscheme
+  qltest:
+    runs-on: ubuntu-latest-xl
+    strategy:
+      fail-fast: false
+      matrix:
+        slice: ["1/2", "2/2"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./csharp/actions/create-extractor-pack
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: csharp-qltest-${{ matrix.slice }}
+      - name: Run QL tests
+        run: |
+          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
   unit-tests:
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest]
+        os: [ubuntu-latest, windows-2019]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
       - name: Setup dotnet
-        uses: actions/setup-dotnet@v4
+        uses: actions/setup-dotnet@v3
         with:
-          dotnet-version: 10.0.100
+          dotnet-version: 7.0.102
       - name: Extractor unit tests
         run: |
-          dotnet tool restore
-          dotnet test -p:RuntimeFrameworkVersion=10.0.0 extractor/Semmle.Util.Tests
-          dotnet test -p:RuntimeFrameworkVersion=10.0.0 extractor/Semmle.Extraction.Tests
-          dotnet test -p:RuntimeFrameworkVersion=10.0.0 autobuilder/Semmle.Autobuild.CSharp.Tests
-          dotnet test -p:RuntimeFrameworkVersion=10.0.0 autobuilder/Semmle.Autobuild.Cpp.Tests
+          dotnet test -p:RuntimeFrameworkVersion=7.0.2 extractor/Semmle.Util.Tests
+          dotnet test -p:RuntimeFrameworkVersion=7.0.2 extractor/Semmle.Extraction.Tests
+          dotnet test -p:RuntimeFrameworkVersion=7.0.2 autobuilder/Semmle.Autobuild.CSharp.Tests
+          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"
         shell: bash
   stubgentest:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
       - uses: ./csharp/actions/create-extractor-pack
       - name: Run stub generator tests
         run: |
@@ -66,6 +96,6 @@ jobs:
           # Update existing stubs in the repo with the freshly generated ones
           mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/
           git status
-          codeql test run --threads=0 --search-path "${{ github.workspace }}" --check-databases --check-diff-informed --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
+          codeql test run --threads=0 --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/csv-coverage-metrics.yml

@@ -14,16 +14,12 @@ on:
       - ".github/workflows/csv-coverage-metrics.yml"
       - ".github/actions/fetch-codeql/action.yml"
 
-permissions:
-  contents: read
-  security-events: write
-
 jobs:
   publish-java:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
       - name: Create empty database
@@ -37,7 +33,7 @@ jobs:
         run: |
           DATABASE="${{ runner.temp }}/java-database"
           codeql database analyze --format=sarif-latest --output=metrics-java.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: metrics-java.sarif
           path: metrics-java.sarif
@@ -51,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
       - name: Create empty database
@@ -64,7 +60,7 @@ jobs:
         run: |
           DATABASE="${{ runner.temp }}/csharp-database"
           codeql database analyze --format=sarif-latest --output=metrics-csharp.sarif -- "$DATABASE" ./csharp/ql/src/Metrics/Summaries/FrameworkCoverage.ql
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: metrics-csharp.sarif
           path: metrics-csharp.sarif

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -19,10 +19,6 @@ on:
       - main
       - "rc/*"
 
-permissions:
-  contents: read
-  pull-requests: read
-
 jobs:
   generate:
     name: Generate framework coverage artifacts
@@ -35,11 +31,11 @@ jobs:
           GITHUB_CONTEXT: ${{ toJSON(github.event) }}
         run: echo "$GITHUB_CONTEXT"
       - name: Clone self (github/codeql) - MERGE
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
         with:
           path: merge
       - name: Clone self (github/codeql) - BASE
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
         with:
           fetch-depth: 2
           path: base
@@ -71,21 +67,21 @@ jobs:
         run: |
           python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
       - name: Upload CSV package list
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: csv-framework-coverage-merge
           path: |
             out_merge/framework-coverage-*.csv
             out_merge/framework-coverage-*.rst
       - name: Upload CSV package list
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: csv-framework-coverage-base
           path: |
             out_base/framework-coverage-*.csv
             out_base/framework-coverage-*.rst
       - name: Upload comparison results
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: comparison
           path: |
@@ -97,7 +93,7 @@ jobs:
         env:
           PR_NUMBER: ${{ github.event.pull_request.number }}
       - name: Upload PR number
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: pr
           path: pr/
@@ -117,7 +113,7 @@ jobs:
           GITHUB_TOKEN: ${{ github.token }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
       - name: Upload comment ID (if it exists)
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: comment
           path: comment/

.github/workflows/csv-coverage-pr-comment.yml

@@ -6,10 +6,6 @@ on:
     types:
       - completed
 
-permissions:
-  contents: read
-  pull-requests: write
-
 jobs:
   check:
     name: Check framework coverage differences and comment
@@ -24,7 +20,7 @@ jobs:
         GITHUB_CONTEXT: ${{ toJSON(github.event) }}
       run: echo "$GITHUB_CONTEXT"
     - name: Clone self (github/codeql)
-      uses: actions/checkout@v5
+      uses: actions/checkout@v4
     - name: Set up Python 3.8
       uses: actions/setup-python@v4
       with:

.github/workflows/csv-coverage-timeseries.yml

@@ -3,20 +3,17 @@ name: Build framework coverage timeseries reports
 on:
   workflow_dispatch:
 
-permissions:
-  contents: read
-
 jobs:
   build:
     runs-on: ubuntu-latest
 
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
         with:
           path: script
       - name: Clone self (github/codeql) for analysis
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
         with:
           path: codeqlModels
           fetch-depth: 0
@@ -30,7 +27,7 @@ jobs:
         run: |
           python script/misc/scripts/library-coverage/generate-timeseries.py codeqlModels
       - name: Upload timeseries CSV
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: framework-coverage-timeseries
           path: framework-coverage-timeseries-*.csv

.github/workflows/csv-coverage-update.yml

@@ -5,10 +5,6 @@ on:
   schedule:
     - cron: "0 0 * * *"
 
-permissions:
-  contents: write
-  pull-requests: write
-
 jobs:
   update:
     name: Update framework coverage report
@@ -21,7 +17,7 @@ jobs:
           GITHUB_CONTEXT: ${{ toJSON(github.event) }}
         run: echo "$GITHUB_CONTEXT"
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
         with:
           path: ql
           fetch-depth: 0

.github/workflows/csv-coverage.yml

@@ -7,20 +7,17 @@ on:
         description: "github/codeql repo SHA used for looking up the CSV models"
         required: false
 
-permissions:
-  contents: read
-
 jobs:
   build:
     runs-on: ubuntu-latest
 
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
         with:
           path: script
       - name: Clone self (github/codeql) for analysis
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
         with:
           path: codeqlModels
           ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}
@@ -34,12 +31,12 @@ jobs:
         run: |
           python script/misc/scripts/library-coverage/generate-report.py ci codeqlModels script
       - name: Upload CSV package list
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: framework-coverage-csv
           path: framework-coverage-*.csv
       - name: Upload RST package list
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: framework-coverage-rst
           path: framework-coverage-*.rst

.github/workflows/fast-forward.yml

@@ -7,14 +7,13 @@ name: Fast-forward tracking branch for selected CodeQL version
 on:
   workflow_dispatch:
 
-permissions:
-  contents: write
-
 jobs:
   fast-forward:
     name: Fast-forward tracking branch for selected CodeQL version
     runs-on: ubuntu-latest
     if: github.repository == 'github/codeql'
+    permissions:
+      contents: write
     env:
       BRANCH_NAME: 'lgtm.com'
     steps:
@@ -26,7 +25,7 @@ jobs:
           exit 1
 
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
 
       - name: Git config
         shell: bash

.github/workflows/go-tests-other-os.yml

@@ -0,0 +1,82 @@
+name: "Go: Run Tests - Other OS"
+on:
+  pull_request:
+    paths:
+      - "go/**"
+      - "!go/ql/**" # don't run other-os if only ql/ files changed
+      - .github/workflows/go-tests-other-os.yml
+      - .github/actions/**
+      - codeql-workspace.yml
+env:
+  GO_VERSION: '~1.21.0'
+jobs:
+  test-mac:
+    name: Test MacOS
+    runs-on: macos-latest
+    steps:
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ env.GO_VERSION }}
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: go-qltest
+      - name: Test
+        run: |
+          cd go
+          make test cache="${{ steps.query-cache.outputs.cache-dir }}"
+
+  test-win:
+    name: Test Windows
+    runs-on: windows-latest-xl
+    steps:
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ env.GO_VERSION }}
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: go-qltest
+
+      - name: Test
+        run: |
+          cd go
+          make test cache="${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/go-tests.yml

@@ -1,29 +1,71 @@
 name: "Go: Run Tests"
 on:
-  pull_request:
+  push:
     paths:
       - "go/**"
-      - "!go/documentation/**"
-      - "shared/**"
       - .github/workflows/go-tests.yml
       - .github/actions/**
       - codeql-workspace.yml
-      - MODULE.bazel
-      - .bazelrc
-      - misc/bazel/**
-
-permissions:
-  contents: read
-
+    branches:
+      - main
+      - "rc/*"
+  pull_request:
+    paths:
+      - "go/**"
+      - .github/workflows/go-tests.yml
+      - .github/actions/**
+      - codeql-workspace.yml
+env:
+  GO_VERSION: '~1.21.0'
 jobs:
   test-linux:
-    if: github.repository_owner == 'github'
     name: Test Linux (Ubuntu)
     runs-on: ubuntu-latest-xl
     steps:
-      - name: Check out code
-        uses: actions/checkout@v5
-      - name: Run tests
-        uses: ./go/actions/test
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v4
         with:
-          run-code-checks: true
+          go-version: ${{ env.GO_VERSION }}
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Check that all Go code is autoformatted
+        run: |
+          cd go
+          make check-formatting
+
+      - name: Compile qhelp files to markdown
+        run: |
+          cd go
+          env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown
+
+      - name: Upload qhelp markdown
+        uses: actions/upload-artifact@v3
+        with:
+          name: qhelp-markdown
+          path: go/qhelp-out/**/*.md
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: go-qltest
+
+      - name: Test
+        run: |
+          cd go
+          make test cache="${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/kotlin-build.yml

@@ -1,28 +0,0 @@
-name: "Kotlin Build"
-
-on:
-  pull_request:
-    paths:
-      - "java/kotlin-extractor/**"
-      - "misc/bazel/**"
-      - "misc/codegen/**"
-      - "*.bazel*"
-      - .github/workflows/kotlin-build.yml
-    branches:
-      - main
-      - rc/*
-      - codeql-cli-*
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v5
-      - run: |
-          bazel query //java/kotlin-extractor/...
-          # only build the default version as a quick check that we can build from `codeql`
-          # the full official build will be checked by QLucie
-          bazel build //java/kotlin-extractor

.github/workflows/labeler.yml

@@ -2,12 +2,11 @@ name: "Pull Request Labeler"
 on:
 - pull_request_target
 
-permissions:
-  contents: read
-  pull-requests: write
-
 jobs:
   triage:
+    permissions:
+      contents: read
+      pull-requests: write
     runs-on: ubuntu-latest
     steps:
     - uses: actions/labeler@v4

.github/workflows/mad_modelDiff.yml

@@ -12,7 +12,6 @@ on:
       - main
     paths:
       - "java/ql/src/utils/modelgenerator/**/*.*"
-      - "misc/scripts/models-as-data/*.*"
       - ".github/workflows/mad_modelDiff.yml"
 
 permissions:
@@ -28,30 +27,24 @@ jobs:
         slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
     steps:
       - name: Clone github/codeql from PR
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
         if: github.event.pull_request
         with:
           path: codeql-pr
       - name: Clone github/codeql from main
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
         with:
           path: codeql-main
           ref: main
       - uses: ./codeql-main/.github/actions/fetch-codeql
-      # compute the shortname of the project that does not contain any special (disk) characters
-      - run: |
-          echo "SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}" >> $GITHUB_OUTPUT
-        env: 
-          SLUG: ${{ matrix.slug }}
-        id: shortname
       - name: Download database
         env:
           SLUG: ${{ matrix.slug }}
           GH_TOKEN: ${{ github.token }}
-          SHORTNAME: ${{ steps.shortname.outputs.SHORTNAME }}
         run: |
           set -x
           mkdir lib-dbs
+          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
           gh api -H "Accept: application/zip" "/repos/${SLUG}/code-scanning/codeql/databases/java" > "$SHORTNAME.zip"
           unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
           mkdir "lib-dbs/$SHORTNAME/"
@@ -68,9 +61,8 @@ jobs:
             DATABASE=$2
             cd codeql-$QL_VARIANT
             SHORTNAME=`basename $DATABASE`
-            python misc/scripts/models-as-data/generate_mad.py --language java --with-summaries --with-sinks $DATABASE $SHORTNAME/$QL_VARIANT
-            mkdir -p $MODELS/$SHORTNAME
-            mv java/ql/lib/ext/generated/modelgenerator/$SHORTNAME/$QL_VARIANT $MODELS/$SHORTNAME
+            python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE ${SHORTNAME}.temp.model.yml
+            mv java/ql/lib/ext/generated/${SHORTNAME}.temp.model.yml $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.model.yml
             cd ..
           }
 
@@ -93,20 +85,20 @@ jobs:
           set -x
           MODELS=`pwd`/tmp-models
           ls -1 tmp-models/
-          for m in $MODELS/*/main/*.model.yml ; do
+          for m in $MODELS/*_main.model.yml ; do
             t="${m/main/"pr"}"
             basename=`basename $m`
-            name="diff_${basename/.model.yml/""}"
+            name="diff_${basename/_main.model.yml/""}"
             (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
           done
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
-          name: models-${{ steps.shortname.outputs.SHORTNAME }}
-          path: tmp-models/**/**/*.model.yml
+          name: models
+          path: tmp-models/*.model.yml
           retention-days: 20
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
-          name: diffs-${{ steps.shortname.outputs.SHORTNAME }}
+          name: diffs
           path: tmp-models/*.html
           # An html file is only produced if the generated models differ.
           if-no-files-found: ignore

.github/workflows/mad_regenerate-models.yml

@@ -11,9 +11,6 @@ on:
       - ".github/workflows/mad_regenerate-models.yml"
       - ".github/actions/fetch-codeql/action.yml"
 
-permissions:
-  contents: read
-
 jobs:
   regenerate-models:
     runs-on: ubuntu-latest
@@ -30,11 +27,11 @@ jobs:
             ref: "placeholder"
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
       - name: Setup CodeQL binaries
         uses: ./.github/actions/fetch-codeql
       - name: Clone repositories
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
         with:
           path: repos/${{ matrix.ref }}
           ref: ${{ matrix.ref }}
@@ -59,7 +56,7 @@ jobs:
           find java -name "*.model.yml" -print0 | xargs -0 git add
           git status
           git diff --cached > models.patch
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: patch
           path: models.patch

.github/workflows/post-pr-comment.yml

@@ -17,11 +17,8 @@ jobs:
   post_comment:
     runs-on: ubuntu-latest
     steps:
-      - name: Download artifacts
-        run: |
-          gh run download "${WORKFLOW_RUN_ID}" --repo "${GITHUB_REPOSITORY}" --name "comment-pr-number"
-          gh run download "${WORKFLOW_RUN_ID}" --repo "${GITHUB_REPOSITORY}" --name "comment-body"
-          gh run download "${WORKFLOW_RUN_ID}" --repo "${GITHUB_REPOSITORY}" --name "comment-id"
+      - name: Download artifact
+        run: gh run download "${WORKFLOW_RUN_ID}" --repo "${GITHUB_REPOSITORY}" --name "comment"
         env:
           GITHUB_TOKEN: ${{ github.token }}
           WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}

.github/workflows/python-tooling.yml

@@ -1,35 +0,0 @@
-name: Python tooling
-
-on:
-  pull_request:
-    paths:
-      - "misc/bazel/**"
-      - "misc/codegen/**"
-      - "misc/scripts/models-as-data/*.py"
-      - "*.bazel*"
-      - .github/workflows/codegen.yml
-      - .pre-commit-config.yaml
-    branches:
-      - main
-      - rc/*
-      - codeql-cli-*
-
-permissions:
-  contents: read
-
-jobs:
-  check-python-tooling:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v5
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
-      - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
-        name: Check that python code is properly formatted
-        with:
-          extra_args: black --all-files
-      - name: Run codegen tests
-        shell: bash
-        run: |
-          bazel test //misc/codegen/...

.github/workflows/qhelp-pr-preview.yml

@@ -36,14 +36,14 @@ jobs:
       - run: echo "${PR_NUMBER}" > pr_number.txt
         env:
           PR_NUMBER: ${{ github.event.number }}
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
-          name: comment-pr-number
+          name: comment
           path: pr_number.txt
           if-no-files-found: error
           retention-days: 1
 
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 2
           persist-credentials: false
@@ -77,10 +77,10 @@ jobs:
           done < "${RUNNER_TEMP}/paths.txt" >> comment_body.txt
           exit "${EXIT_CODE}"
 
-      - if: ${{ !cancelled() }}
-        uses: actions/upload-artifact@v4
+      - if: always()
+        uses: actions/upload-artifact@v3
         with:
-          name: comment-body
+          name: comment
           path: comment_body.txt
           if-no-files-found: error
           retention-days: 1
@@ -94,9 +94,9 @@ jobs:
           GITHUB_TOKEN: ${{ github.token }}
           PR_NUMBER: ${{ github.event.number }}
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
-          name: comment-id
+          name: comment
           path: comment_id.txt
           if-no-files-found: error
           retention-days: 1

.github/workflows/ql-for-ql-build.yml

@@ -9,25 +9,19 @@ on:
 env:
   CARGO_TERM_COLOR: always
 
-permissions:
-  contents: read
-  security-events: write
-
 jobs:
   analyze:
-    if: github.repository_owner == 'github'
     runs-on: ubuntu-latest-xl
     steps:
       ### Build the queries ###
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@main
+        uses: github/codeql-action/init@v2
         with:
           languages: javascript # does not matter
-          tools: nightly
       - uses: ./.github/actions/os-version
         id: os_version
       ### Build the extractor ###
@@ -50,20 +44,20 @@ jobs:
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Release build
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; ./scripts/create-extractor-pack.sh
+        run: cd ql; ./scripts/create-extractor-pack.sh       
         env:
-          GH_TOKEN: ${{ github.token }}
+          GH_TOKEN: ${{ github.token }}   
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with:
+        with: 
           key: run-ql-for-ql
       - name: Make database and analyze
         run: |
           ./ql/target/release/buramu | tee deprecated.blame # Add a blame file for the extractor to parse.
-          ${CODEQL} database create -l=ql ${DB} --search-path "${{ github.workspace }}"
+          ${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB}
           ${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls  --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-        env:
+        env: 
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
           DB: ${{ runner.temp }}/DB
           LGTM_INDEX_FILTERS: |
@@ -71,12 +65,12 @@ jobs:
             exclude:*/ql/lib/upgrades/
             exclude:java/ql/integration-tests
       - name: Upload sarif to code-scanning
-        uses: github/codeql-action/upload-sarif@main
+        uses: github/codeql-action/upload-sarif@v2
         with:
           sarif_file: ql-for-ql.sarif
           category: ql-for-ql
       - name: Sarif as artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: ql-for-ql.sarif
           path: ql-for-ql.sarif
@@ -85,7 +79,7 @@ jobs:
           mkdir split-sarif
           node ./ql/scripts/split-sarif.js ql-for-ql.sarif split-sarif
       - name: Upload langs as artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: ql-for-ql-langs
           path: split-sarif

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -11,10 +11,6 @@ on:
       - ql/ql/src/ql.dbscheme
   workflow_dispatch:
 
-permissions:
-  contents: read
-  security-events: read
-
 jobs:
   measure:
     env:
@@ -25,11 +21,11 @@ jobs:
           - github/codeql
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@main
+        uses: github/codeql-action/init@v2
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version
@@ -46,15 +42,15 @@ jobs:
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
         with:
           repository: ${{ matrix.repo }}
           path: ${{ github.workspace }}/repo
       - name: Create database
         run: |
           "${CODEQL}" database create \
-          --search-path "${{ github.workspace }}" \
-          --threads 4 \
+            --search-path "ql/extractor-pack" \
+            --threads 4 \
             --language ql --source-root "${{ github.workspace }}/repo" \
             "${{ runner.temp }}/database"
         env:
@@ -65,7 +61,7 @@ jobs:
           "${CODEQL}" dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ql"
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: measurements
           path: stats
@@ -75,15 +71,15 @@ jobs:
     runs-on: ubuntu-latest
     needs: measure
     steps:
-      - uses: actions/checkout@v5
-      - uses: actions/download-artifact@v4
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v3
         with:
           name: measurements
           path: stats
       - run: |
           python -m pip install --user lxml
           find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ruby/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: ql.dbscheme.stats
           path: ql/ql/src/ql.dbscheme.stats

.github/workflows/ql-for-ql-tests.yml

@@ -17,20 +17,16 @@ on:
 env:
   CARGO_TERM_COLOR: always
 
-permissions:
-  contents: read
-
 jobs:
   qltest:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@main
+        uses: github/codeql-action/init@v2
         with:
           languages: javascript # does not matter
-          tools: nightly
       - uses: ./.github/actions/os-version
         id: os_version
       - uses: actions/cache@v3
@@ -41,7 +37,7 @@ jobs:
             ql/target
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/rust-toolchain.toml', 'ql/**/Cargo.lock') }}
       - name: Check formatting
-        run: cd ql; cargo fmt -- --check
+        run: cd ql; cargo fmt --all -- --check
       - name: Build extractor
         run: |
           cd ql;
@@ -50,33 +46,32 @@ jobs:
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with:
+        with: 
           key: ql-for-ql-tests
       - name: Run QL tests
         run: |
-          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
+          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
 
-  other-os:
+  other-os: 
     strategy:
       matrix:
         os: [macos-latest, windows-latest]
     needs: [qltest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v5
-      - name: Install GNU tar
+      - uses: actions/checkout@v4
+      - name: Install GNU tar 
         if: runner.os == 'macOS'
         run: |
           brew install gnu-tar
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@main
+        uses: github/codeql-action/init@v2
         with:
           languages: javascript # does not matter
-          tools: nightly
       - uses: ./.github/actions/os-version
         id: os_version
       - uses: actions/cache@v3
@@ -102,7 +97,7 @@ jobs:
       - name: Run a single QL tests - Unix
         if: runner.os != 'Windows'
         run: |
-          "${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
+          "${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Run a single QL tests - Windows
@@ -110,4 +105,5 @@ jobs:
         shell: pwsh
         run: |
           $Env:PATH += ";$(dirname ${{ steps.find-codeql.outputs.codeql-path }})"
-          codeql test run --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
+          codeql test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
+      

.github/workflows/query-list.yml

@@ -13,9 +13,6 @@ on:
       - '.github/actions/fetch-codeql/action.yml'
       - 'misc/scripts/generate-code-scanning-query-list.py'
 
-permissions:
-  contents: read
-
 jobs:
   build:
 
@@ -23,7 +20,7 @@ jobs:
 
     steps:
     - name: Clone self (github/codeql)
-      uses: actions/checkout@v5
+      uses: actions/checkout@v4
       with:
         path: codeql 
     - name: Set up Python 3.8
@@ -31,13 +28,13 @@ jobs:
       with:
         python-version: 3.8
     - name: Download CodeQL CLI
-      # Look under the `codeql` directory, as this is where we checked out the `github/codeql` repo
+      # Look under the `codeql` directory, as this is where we checked out the `github/codeql` repo
       uses: ./codeql/.github/actions/fetch-codeql
     - name: Build code scanning query list
       run: |
         python codeql/misc/scripts/generate-code-scanning-query-list.py > code-scanning-query-list.csv
     - name: Upload code scanning query list
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v3
       with:
         name: code-scanning-query-list
         path: code-scanning-query-list.csv

.github/workflows/ruby-build.yml

@@ -0,0 +1,284 @@
+name: "Ruby: Build"
+
+on:
+  push:
+    paths:
+      - "ruby/**"
+      - .github/workflows/ruby-build.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+      - "rc/*"
+  pull_request:
+    paths:
+      - "ruby/**"
+      - .github/workflows/ruby-build.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+      - "rc/*"
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Version tag to create"
+        required: false
+
+env:
+  CARGO_TERM_COLOR: always
+
+defaults:
+  run:
+    working-directory: ruby
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install GNU tar
+        if: runner.os == 'macOS'
+        run: |
+          brew install gnu-tar
+          echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
+      - name: Install cargo-cross
+        if: runner.os == 'Linux'
+        run: cargo install cross --version 0.2.5
+      - uses: ./.github/actions/os-version
+        id: os_version
+      - name: Cache entire extractor
+        uses: actions/cache@v3
+        id: cache-extractor
+        with:
+          path: |
+            ruby/extractor/target/release/codeql-extractor-ruby
+            ruby/extractor/target/release/codeql-extractor-ruby.exe
+            ruby/extractor/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ruby/extractor/**/*.rs') }}
+      - uses: actions/cache@v3
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            ruby/target
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-rust-cargo-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/**/Cargo.lock') }}
+      - name: Check formatting
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        run: cd extractor && cargo fmt --all -- --check
+      - name: Build
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        run: cd extractor && cargo build --verbose
+      - name: Run tests
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        run: cd extractor && cargo test --verbose
+      # On linux, build the extractor via cross in a centos7 container.
+      # This ensures we don't depend on glibc > 2.17.
+      - name: Release build (linux)
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os == 'Linux'
+        run: |
+          cd extractor
+          cross build --release
+          mv target/x86_64-unknown-linux-gnu/release/codeql-extractor-ruby target/release/
+      - name: Release build (windows and macos)
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os != 'Linux'
+        run: cd extractor && cargo build --release
+      - name: Generate dbscheme
+        if: ${{ matrix.os == 'ubuntu-latest' && steps.cache-extractor.outputs.cache-hit != 'true'}}
+        run: extractor/target/release/codeql-extractor-ruby generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+      - uses: actions/upload-artifact@v3
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        with:
+          name: ruby.dbscheme
+          path: ruby/ql/lib/ruby.dbscheme
+      - uses: actions/upload-artifact@v3
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        with:
+          name: TreeSitter.qll
+          path: ruby/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+      - uses: actions/upload-artifact@v3
+        with:
+          name: extractor-${{ matrix.os }}
+          path: |
+            ruby/extractor/target/release/codeql-extractor-ruby
+            ruby/extractor/target/release/codeql-extractor-ruby.exe
+          retention-days: 1
+  compile-queries:
+    runs-on: ubuntu-latest-xl
+    steps:
+      - uses: actions/checkout@v4
+      - name: Fetch CodeQL
+        uses: ./.github/actions/fetch-codeql
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: ruby-build
+      - name: Build Query Pack
+        run: |
+          PACKS=${{ runner.temp }}/query-packs
+          rm -rf $PACKS
+          codeql pack create ../misc/suite-helpers --output "$PACKS"
+          codeql pack create ../shared/regex --output "$PACKS"
+          codeql pack create ../shared/ssa --output "$PACKS"
+          codeql pack create ../shared/tutorial --output "$PACKS"
+          codeql pack create ql/lib --output "$PACKS"
+          codeql pack create -j0 ql/src --output "$PACKS" --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          PACK_FOLDER=$(readlink -f "$PACKS"/codeql/ruby-queries/*)
+          codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
+          (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
+      - uses: actions/upload-artifact@v3
+        with:
+          name: codeql-ruby-queries
+          path: |
+            ${{ runner.temp }}/query-packs/*
+          retention-days: 1
+
+  package:
+    runs-on: ubuntu-latest
+    needs: [build, compile-queries]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v3
+        with:
+          name: ruby.dbscheme
+          path: ruby/ruby
+      - uses: actions/download-artifact@v3
+        with:
+          name: extractor-ubuntu-latest
+          path: ruby/linux64
+      - uses: actions/download-artifact@v3
+        with:
+          name: extractor-windows-latest
+          path: ruby/win64
+      - uses: actions/download-artifact@v3
+        with:
+          name: extractor-macos-latest
+          path: ruby/osx64
+      - run: |
+          mkdir -p ruby
+          cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
+          mkdir -p ruby/tools/{linux64,osx64,win64}
+          cp linux64/codeql-extractor-ruby ruby/tools/linux64/extractor
+          cp osx64/codeql-extractor-ruby ruby/tools/osx64/extractor
+          cp win64/codeql-extractor-ruby.exe ruby/tools/win64/extractor.exe
+          chmod +x ruby/tools/{linux64,osx64}/extractor
+          zip -rq codeql-ruby.zip ruby
+      - uses: actions/upload-artifact@v3
+        with:
+          name: codeql-ruby-pack
+          path: ruby/codeql-ruby.zip
+          retention-days: 1
+      - uses: actions/download-artifact@v3
+        with:
+          name: codeql-ruby-queries
+          path: ruby/qlpacks
+      - run: |
+          echo '{
+            "provide": [
+            "ruby/codeql-extractor.yml",
+            "qlpacks/*/*/*/qlpack.yml"
+            ]
+          }' > .codeqlmanifest.json
+          zip -rq codeql-ruby-bundle.zip .codeqlmanifest.json ruby qlpacks
+      - uses: actions/upload-artifact@v3
+        with:
+          name: codeql-ruby-bundle
+          path: ruby/codeql-ruby-bundle.zip
+          retention-days: 1
+
+  test:
+    defaults:
+      run:
+        working-directory: ${{ github.workspace }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    runs-on: ${{ matrix.os }}
+    needs: [package]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Fetch CodeQL
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Download Ruby bundle
+        uses: actions/download-artifact@v3
+        with:
+          name: codeql-ruby-bundle
+          path: ${{ runner.temp }}
+      - name: Unzip Ruby bundle
+        shell: bash
+        run: unzip -q -d "${{ runner.temp }}/ruby-bundle" "${{ runner.temp }}/codeql-ruby-bundle.zip"
+
+      - name: Run QL test
+        shell: bash
+        run: |
+          codeql test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" ruby/ql/test/library-tests/ast/constants/
+      - name: Create database
+        shell: bash
+        run: |
+          codeql database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root ruby/ql/test/library-tests/ast/constants/ ../database
+      - name: Analyze database
+        shell: bash
+        run: |
+          codeql database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls
+
+  # This is a copy of the 'test' job that runs in a centos7 container.
+  # This tests that the extractor works correctly on systems with an old glibc.
+  test-centos7:
+    defaults:
+      run:
+        working-directory: ${{ github.workspace }}
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    container:
+      image: centos:centos7
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    needs: [package]
+    steps:
+      - name: Install gh cli
+        run: |
+          yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
+          # fetch-codeql requires unzip and jq
+          # jq is available in epel-release (https://docs.fedoraproject.org/en-US/epel/)
+          yum install -y gh unzip epel-release
+          yum install -y jq
+      - uses: actions/checkout@v3
+      - name: Fetch CodeQL
+        uses: ./.github/actions/fetch-codeql
+
+      # Due to a bug in Actions, we can't use runner.temp in the run blocks here.
+      # https://github.com/actions/runner/issues/2185
+
+      - name: Download Ruby bundle
+        uses: actions/download-artifact@v3
+        with:
+          name: codeql-ruby-bundle
+          path: ${{ runner.temp }}
+      - name: Unzip Ruby bundle
+        shell: bash
+        run: unzip -q -d "$RUNNER_TEMP"/ruby-bundle "$RUNNER_TEMP"/codeql-ruby-bundle.zip
+
+      - name: Run QL test
+        shell: bash
+        run: |
+          codeql test run --search-path "$RUNNER_TEMP"/ruby-bundle --additional-packs "$RUNNER_TEMP"/ruby-bundle ruby/ql/test/library-tests/ast/constants/
+      - name: Create database
+        shell: bash
+        run: |
+          codeql database create --search-path "$RUNNER_TEMP"/ruby-bundle --language ruby --source-root ruby/ql/test/library-tests/ast/constants/ ../database
+      - name: Analyze database
+        shell: bash
+        run: |
+          codeql database analyze --search-path "$RUNNER_TEMP"/ruby-bundle --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/ruby-dataset-measure.yml

@@ -0,0 +1,73 @@
+name: "Ruby: Collect database stats"
+
+on:
+  push:
+    branches:
+      - main
+      - "rc/*"
+    paths:
+      - ruby/ql/lib/ruby.dbscheme
+      - .github/workflows/ruby-dataset-measure.yml
+  pull_request:
+    branches:
+      - main
+      - "rc/*"
+    paths:
+      - ruby/ql/lib/ruby.dbscheme
+      - .github/workflows/ruby-dataset-measure.yml
+  workflow_dispatch:
+
+jobs:
+  measure:
+    env:
+      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
+    strategy:
+      fail-fast: false
+      matrix:
+        repo: [rails/rails, discourse/discourse, spree/spree, ruby/ruby]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - uses: ./ruby/actions/create-extractor-pack
+
+      - name: Checkout ${{ matrix.repo }}
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ matrix.repo }}
+          path: ${{ github.workspace }}/repo
+      - name: Create database
+        run: |
+          codeql database create \
+            --search-path "${{ github.workspace }}/ruby/extractor-pack" \
+            --threads 4 \
+            --language ruby --source-root "${{ github.workspace }}/repo" \
+            "${{ runner.temp }}/database"
+      - name: Measure database
+        run: |
+          mkdir -p "stats/${{ matrix.repo }}"
+          codeql dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ruby"
+      - uses: actions/upload-artifact@v3
+        with:
+          name: measurements
+          path: stats
+          retention-days: 1
+
+  merge:
+    runs-on: ubuntu-latest
+    needs: measure
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v3
+        with:
+          name: measurements
+          path: stats
+      - run: |
+          python -m pip install --user lxml
+          find stats -name 'stats.xml' | sort | xargs python ruby/scripts/merge_stats.py --output ruby/ql/lib/ruby.dbscheme.stats --normalise ruby_tokeninfo
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ruby.dbscheme.stats
+          path: ruby/ql/lib/ruby.dbscheme.stats

.github/workflows/ruby-qltest.yml

@@ -0,0 +1,69 @@
+name: "Ruby: Run QL Tests"
+
+on:
+  push:
+    paths:
+      - "ruby/**"
+      - "shared/**"
+      - .github/workflows/ruby-build.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+      - "rc/*"
+  pull_request:
+    paths:
+      - "ruby/**"
+      - "shared/**"
+      - .github/workflows/ruby-qltest.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+      - "rc/*"
+
+env:
+  CARGO_TERM_COLOR: always
+
+defaults:
+  run:
+    working-directory: ruby
+
+jobs:
+  qlupgrade:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/fetch-codeql
+      - name: Check DB upgrade scripts
+        run: |
+          echo >empty.trap
+          codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
+          codeql dataset upgrade testdb --additional-packs ql/lib
+          diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme
+      - name: Check DB downgrade scripts
+        run: |
+          echo >empty.trap
+          rm -rf testdb; codeql dataset import -S ql/lib/ruby.dbscheme testdb empty.trap
+          codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
+           --dbscheme=ql/lib/ruby.dbscheme --target-dbscheme=downgrades/initial/ruby.dbscheme |
+           xargs codeql execute upgrades testdb
+          diff -q testdb/ruby.dbscheme downgrades/initial/ruby.dbscheme
+  qltest:
+    runs-on: ubuntu-latest-xl
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/fetch-codeql
+      - uses: ./ruby/actions/create-extractor-pack
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: ruby-qltest
+      - name: Run QL tests
+        run: |
+          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/rust-analysis.yml

@@ -1,64 +0,0 @@
-name: "Code scanning - Rust"
-
-on:
-  push:
-    branches:
-      - main
-      - 'rc/*'
-  pull_request:
-    branches:
-      - main
-      - 'rc/*'
-    paths:
-      - '**/*.rs'
-      - '**/Cargo.toml'
-      - '.github/codeql/codeql-config.yml'
-      - '.github/workflows/rust-analysis.yml'
-  schedule:
-    - cron: '0 9 * * 1'
-
-env:
-  CODEQL_ENABLE_EXPERIMENTAL_FEATURES: "true"
-
-jobs:
-  analyze:
-    strategy:
-      matrix:
-        language: [ 'rust' ]
-
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: read
-      security-events: write
-      pull-requests: read
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v5
-
-    - name: Query latest nightly CodeQL bundle
-      shell: bash
-      id: codeql
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-      run: |
-        REPO=dsp-testing/codeql-cli-nightlies
-        TAG=$(
-          gh release list -R $REPO -L1 --exclude-drafts --json tagName -q ".[] | .tagName"
-        )
-        echo "nightly_bundle=https://github.com/$REPO/releases/download/$TAG/codeql-bundle-linux64.tar.zst" \
-          | tee -a "$GITHUB_OUTPUT"
-
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@main
-      with:
-        tools: ${{ steps.codeql.outputs.nightly_bundle }}
-        languages: ${{ matrix.language }}
-        config-file: ./.github/codeql/codeql-config.yml
-
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@main
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@main

.github/workflows/rust.yml

@@ -1,80 +0,0 @@
-name: "Rust"
-
-on:
-  pull_request:
-    paths:
-      - "rust/**"
-      - "misc/bazel/**"
-      - "misc/codegen/**"
-      - "shared/**"
-      - "MODULE.bazel"
-      - .github/workflows/rust.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-      - "!**/*.md"
-      - "!**/*.qhelp"
-    branches:
-      - rust-experiment
-      - main
-      - rc/*
-      - codeql-cli-*
-
-permissions:
-  contents: read
-
-jobs:
-  rust-ast-generator:
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: rust/ast-generator
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v5
-      - name: Inject sources
-        shell: bash
-        run: |
-          bazel run //rust/ast-generator:inject-sources
-      - name: Format
-        shell: bash
-        run: |
-          cargo fmt --check
-      - name: Compilation
-        shell: bash
-        run: cargo check
-      - name: Clippy
-        shell: bash
-        run: |
-          cargo clippy --no-deps -- -D warnings
-  rust-code:
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: rust/extractor
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v5
-      - name: Format
-        shell: bash
-        run: |
-          cargo fmt --check
-      - name: Compilation
-        shell: bash
-        run: cargo check
-      - name: Clippy
-        shell: bash
-        run: |
-          cargo clippy --no-deps -- -D warnings
-  rust-codegen:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v5
-      - name: Install CodeQL
-        uses: ./.github/actions/fetch-codeql
-      - name: Code generation
-        shell: bash
-        run: |
-          bazel run //rust/codegen
-          git add .
-          git diff --exit-code HEAD

.github/workflows/swift.yml

@@ -6,7 +6,6 @@ on:
       - "swift/**"
       - "misc/bazel/**"
       - "misc/codegen/**"
-      - "shared/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**
@@ -18,65 +17,91 @@ on:
       - main
       - rc/*
       - codeql-cli-*
-
-permissions:
-  contents: read
-
-defaults:
-  run:
-    shell: bash
-    working-directory: swift
+  push:
+    paths:
+      - "swift/**"
+      - "misc/bazel/**"
+      - "misc/codegen/**"
+      - "*.bazel*"
+      - .github/workflows/swift.yml
+      - .github/actions/**
+      - codeql-workspace.yml
+      - "!**/*.md"
+      - "!**/*.qhelp"
+    branches:
+      - main
+      - rc/*
+      - codeql-cli-*
 
 jobs:
-  build-and-test:
-    if: github.repository_owner == 'github'
-    strategy:
-      matrix:
-        runner: [ubuntu-latest, macos-15-xlarge]
-      fail-fast: false
-    runs-on: ${{ matrix.runner }}
+  # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks
+  # without waiting for the macOS build
+  build-and-test-macos:
+    runs-on: macos-12-xl
     steps:
-      - uses: actions/checkout@v5
-      - name: Setup (Linux)
-        if: runner.os == 'Linux'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y uuid-dev zlib1g-dev
-      - name: Build Swift extractor
-        shell: bash
-        run: |
-          bazel run :install
-      - name: Run Swift tests
-        shell: bash
-        run: |
-          bazel test ... --test_tag_filters=-override --test_output=errors
-  clang-format:
+      - uses: actions/checkout@v4
+      - uses: ./swift/actions/build-and-test
+  build-and-test-linux:
+    runs-on: ubuntu-latest-xl
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./swift/actions/build-and-test
+  qltests-linux:
+    needs: build-and-test-linux
+    runs-on: ubuntu-latest-xl
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./swift/actions/run-ql-tests
+  qltests-macos:
+    if : ${{ github.event_name == 'pull_request' }}
+    needs: build-and-test-macos
+    runs-on: macos-12-xl
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./swift/actions/run-ql-tests
+  integration-tests-linux:
+    needs: build-and-test-linux
+    runs-on: ubuntu-latest-xl
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./swift/actions/run-integration-tests
+  integration-tests-macos:
+    if : ${{ github.event_name == 'pull_request' }}
+    needs: build-and-test-macos
+    runs-on: macos-12-xl
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./swift/actions/run-integration-tests
+  codegen:
+    if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
+      - uses: actions/checkout@v4
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version-file: 'swift/.python-version'
+      - uses: pre-commit/action@v3.0.0
         name: Check that python code is properly formatted
         with:
-          extra_args: clang-format --all-files
-  codegen:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v5
+          extra_args: autopep8 --all-files
       - uses: ./.github/actions/fetch-codeql
-      - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
+      - uses: pre-commit/action@v3.0.0
         name: Check that QL generated code was checked in
         with:
           extra_args: swift-codegen --all-files
       - name: Generate C++ files
         run: |
-          bazel run codegen -- --generate=trap,cpp --cpp-output=$PWD/generated-cpp-files
-      - uses: actions/upload-artifact@v4
+          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/generated-cpp-files
+      - uses: actions/upload-artifact@v3
         with:
           name: swift-generated-cpp-files
           path: generated-cpp-files/**
-  check-no-override:
+  database-upgrade-scripts:
+    if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - name: Check that no override is present in load.bzl
-        run: bazel test ... --test_tag_filters=override --test_output=errors
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/fetch-codeql
+      - uses: ./swift/actions/database-upgrade-scripts

.github/workflows/sync-files.yml

@@ -10,14 +10,11 @@ on:
       - main
       - 'rc/*'
 
-permissions:
-  contents: read
-
 jobs:
   sync:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
       - name: Check synchronized files
         run: python config/sync-files.py
       - name: Check dbscheme fragments

.github/workflows/tree-sitter-extractor-test.yml

@@ -23,27 +23,24 @@ defaults:
   run:
     working-directory: shared/tree-sitter-extractor
 
-permissions:
-  contents: read
-
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
       - name: Check formatting
-        run: cargo fmt -- --check
+        run: cargo fmt --all -- --check
       - name: Run tests
         run: cargo test --verbose
   fmt:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest  
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
       - name: Check formatting
         run: cargo fmt --check
   clippy:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest  
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
       - name: Run clippy
         run: cargo clippy -- --no-deps -D warnings -A clippy::new_without_default -A clippy::too_many_arguments

.github/workflows/validate-change-notes.yml

@@ -15,15 +15,12 @@ on:
       - ".github/workflows/validate-change-notes.yml"
       - ".github/actions/fetch-codeql/action.yml"
 
-permissions:
-  contents: read
-
 jobs:
   check-change-note:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v4
 
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
@@ -31,4 +28,4 @@ jobs:
       - name: Fail if there are any errors with existing change notes
 
         run: |
-          codeql pack release --groups actions,cpp,csharp,go,java,javascript,python,ruby,shared,swift -examples,-test,-experimental
+          codeql pack release --groups cpp,csharp,java,javascript,python,ruby,-examples,-test,-experimental

.github/workflows/zipmerge-test.yml

@@ -1,23 +0,0 @@
-name: "Test zipmerge code"
-
-on:
-  pull_request:
-    paths:
-      - "misc/bazel/internal/zipmerge/**"
-      - "MODULE.bazel"
-      - ".bazelrc*"
-    branches:
-      - main
-      - "rc/*"
-
-permissions:
-  contents: read
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v5
-      - run: |
-          bazel test //misc/bazel/internal/zipmerge:test --test_output=all

.gitignore

@@ -7,8 +7,8 @@
 .cache
 
 # qltest projects and artifacts
-*.actual
-*/ql/test*/**/*.testproj
+*/ql/test/**/*.testproj
+*/ql/test/**/*.actual
 */ql/test/**/go.sum
 
 # Visual studio temporaries, except a file used by QL4VS
@@ -39,9 +39,6 @@
 # local bazel options
 /local.bazelrc
 
-# generated cmake directory
-/.bazel-cmake
-
 # CLion project files
 /.clwb
 
@@ -62,20 +59,3 @@ node_modules/
 
 # Temporary folders for working with generated models
 .model-temp
-/mad-generation-build
-
-# bazel-built in-tree extractor packs
-/*/extractor-pack
-
-# Jetbrains IDE files
-.idea
-
-# cargo build directory
-/target
-
-# some upgrade/downgrade checks create these files
-**/upgrades/*/*.dbscheme.stats
-**/downgrades/*/*.dbscheme.stats
-
-# Mergetool files
-*.orig

.lfsconfig

@@ -1,7 +0,0 @@
-[lfs]
-# codeql is publicly forked by many users, and we don't want any LFS file polluting their working
-# copies. We therefore exclude everything by default.
-# For files required by bazel builds, use rules in `misc/bazel/lfs.bzl` to download them on demand.
-# we go for `fetchinclude` to something not exsiting rather than `fetchexclude = *` because the
-# former is easier to override (with `git -c` or a local git config) to fetch something specific
-fetchinclude = /nothing

.pre-commit-config.yaml

@@ -1,44 +1,33 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
-default_language_version:
-  python: python3.12
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.2.0
     hooks:
       - id: trailing-whitespace
-        exclude: /test([^/]*)/.*$(?<!\.qlref)|.*\.patch$|.*\.qll?$
+        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)|.*\.patch
       - id: end-of-file-fixer
-        exclude: Cargo.lock$|/test([^/]*)/.*$(?<!\.qlref)|.*\.patch$|.*\.qll?$
+        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)|.*\.patch
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v17.0.6
+    rev: v13.0.1
     hooks:
       - id: clang-format
+        files: ^swift/.*\.(h|c|cpp)$
 
-  - repo: https://github.com/psf/black
-    rev: 25.1.0
+  - repo: https://github.com/pre-commit/mirrors-autopep8
+    rev: v1.6.0
     hooks:
-      - id: black
-        files: ^(misc/codegen/.*|misc/scripts/models-as-data/.*)\.py$
+      - id: autopep8
+        files: ^misc/codegen/.*\.py
+
+  - repo: https://github.com/warchant/pre-commit-buildifier
+    rev: 0.0.2
+    hooks:
+      - id: buildifier
 
   - repo: local
     hooks:
-      - id: buildifier
-        name: Format bazel files
-        files: \.(bazel|bzl)
-        language: system
-        entry: bazel run //misc/bazel/buildifier
-        pass_filenames: false
-
-#      DISABLED: can be enabled by copying this config and installing `pre-commit` with `--config` on the copy
-#      - id: go-gen
-#        name: Check checked in generated files in go
-#        files: ^go/.*
-#        language: system
-#        entry: bazel run //go:gen
-#        pass_filenames: false
-
       - id: codeql-format
         name: Fix QL file formatting
         files: \.qll?$
@@ -47,7 +36,7 @@ repos:
 
       - id: sync-files
         name: Fix files required to be identical
-        files: \.(qll?|qhelp|swift|toml)$|^config/identical-files\.json$
+        files: \.(qll?|qhelp|swift)$|^config/identical-files\.json$
         language: system
         entry: python3 config/sync-files.py --latest
         pass_filenames: false
@@ -60,7 +49,7 @@ repos:
 
       - id: swift-codegen
         name: Run Swift checked in code generation
-        files: ^misc/codegen/|^swift/(schema.py$|codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements)|ql/\.generated.list)
+        files: ^swift/(schema.py$|codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements)|ql/\.generated.list)
         language: system
         entry: bazel run //swift/codegen -- --quiet
         pass_filenames: false
@@ -71,17 +60,3 @@ repos:
         language: system
         entry: bazel test //misc/codegen/test
         pass_filenames: false
-
-      - id: rust-codegen
-        name: Run Rust checked in code generation
-        files: ^misc/codegen/|^rust/(prefix\.dbscheme|schema/|codegen/|.*/generated/|ql/lib/(rust\.dbscheme$|codeql/rust/elements)|\.generated.list|ast-generator/)
-        language: system
-        entry: bazel run //rust/codegen -- --quiet
-        pass_filenames: false
-
-      - id: rust-lint
-        name: Run fmt and clippy on Rust code
-        files: ^rust/extractor/(.*rs|Cargo.toml)$
-        language: system
-        entry: python3 rust/lint.py
-        pass_filenames: false

.vscode/settings.json

@@ -1,6 +1,5 @@
 {
   "omnisharp.autoStart": false,
   "cmake.sourceDirectory": "${workspaceFolder}/swift",
-  "cmake.buildDirectory": "${workspaceFolder}/bazel-cmake-build",
-  "editor.suggest.matchOnWordStartOnly": false
+  "cmake.buildDirectory": "${workspaceFolder}/bazel-cmake-build"
 }

.vscode/tasks.json

@@ -38,104 +38,6 @@
                 "command": "${config:python.pythonPath}",
             },
             "problemMatcher": []
-        },
-        {
-            "label": "Create query change note",
-            "type": "process",
-            "command": "python3",
-            "args": [
-                "misc/scripts/create-change-note.py",
-                "${input:language}",
-                "src",
-                "${input:name}",
-                "${input:categoryQuery}"
-            ],
-            "options": {
-                "env": {
-                    "EDITOR": "code -r",
-                }
-            },
-            "presentation": {
-                "reveal": "never",
-                "close": true
-            },
-            "problemMatcher": []
-        },
-                {
-            "label": "Create library change note",
-            "type": "process",
-            "command": "python3",
-            "args": [
-                "misc/scripts/create-change-note.py",
-                "${input:language}",
-                "lib",
-                "${input:name}",
-                "${input:categoryLibrary}"
-            ],
-            "options": {
-                "env": {
-                    "EDITOR": "code -r"
-                }
-            },
-            "presentation": {
-                "reveal": "never",
-                "close": true
-            },
-            "problemMatcher": []
-        }
-    ],
-    "inputs": [
-        {
-            "type": "pickString",
-            "id": "language",
-            "description": "Language",
-            "options":
-            [
-                "actions",
-                "go",
-                "java",
-                "javascript",
-                "cpp",
-                "csharp",
-                "python",
-                "ruby",
-                "rust",
-                "swift",
-            ]
-        },
-        {
-            "type": "promptString",
-            "id": "name",
-            "description": "Short name (kebab-case)"
-        },
-        {
-            "type": "pickString",
-            "id": "categoryQuery",
-            "description": "Category (query change)",
-            "options":
-            [
-                "breaking",
-                "deprecated",
-                "newQuery",
-                "queryMetadata",
-                "majorAnalysis",
-                "minorAnalysis",
-                "fix",
-            ]
-        },
-                {
-            "type": "pickString",
-            "id": "categoryLibrary",
-            "description": "Category (library change)",
-            "options":
-            [
-                "breaking",
-                "deprecated",
-                "feature",
-                "majorAnalysis",
-                "minorAnalysis",
-                "fix",
-            ]
         }
     ]
 }

BUILD.bazel

@@ -1,5 +0,0 @@
-exports_files([
-    "LICENSE",
-    "Cargo.lock",
-    "Cargo.toml",
-])

CODEOWNERS

@@ -1,39 +1,16 @@
-# Catch-all for anything which isn't matched by a line lower down
-* @github/code-scanning-alert-coverage
-
-# CodeQL language libraries
-/actions/ @github/codeql-dynamic
 /cpp/ @github/codeql-c-analysis
 /csharp/ @github/codeql-csharp
-/csharp/autobuilder/Semmle.Autobuild.Cpp @github/codeql-c-extractor @github/code-scanning-language-coverage
-/csharp/autobuilder/Semmle.Autobuild.Cpp.Tests @github/codeql-c-extractor @github/code-scanning-language-coverage
 /go/ @github/codeql-go
-/go/codeql-tools/ @github/codeql-go @github/code-scanning-language-coverage
-/go/downgrades/ @github/codeql-go @github/code-scanning-language-coverage
-/go/extractor/ @github/codeql-go @github/code-scanning-language-coverage
-/go/extractor-smoke-test/ @github/codeql-go @github/code-scanning-language-coverage
-/go/ql/test/extractor-tests/ @github/codeql-go @github/code-scanning-language-coverage
 /java/ @github/codeql-java
 /javascript/ @github/codeql-javascript
-/javascript/extractor/ @github/codeql-javascript @github/code-scanning-language-coverage
 /python/ @github/codeql-python
-/python/extractor/ @github/codeql-python @github/code-scanning-language-coverage
-/ql/ @github/codeql-ql-for-ql-reviewers
 /ruby/ @github/codeql-ruby
-/ruby/extractor/ @github/codeql-ruby @github/code-scanning-language-coverage
-/rust/ @github/codeql-rust
-/rust/extractor/ @github/codeql-rust @github/code-scanning-language-coverage
-/shared/ @github/codeql-shared-libraries-reviewers
 /swift/ @github/codeql-swift
-/swift/extractor/ @github/codeql-swift @github/code-scanning-language-coverage
 /misc/codegen/ @github/codeql-swift
-/java/kotlin-extractor/ @github/codeql-kotlin @github/code-scanning-language-coverage
-/java/ql/test-kotlin1/ @github/codeql-kotlin
-/java/ql/test-kotlin2/ @github/codeql-kotlin
+/java/kotlin-extractor/ @github/codeql-kotlin
 
-# Experimental CodeQL cryptography
-**/experimental/**/quantum/ @github/ps-codeql
-/shared/quantum/ @github/ps-codeql
+# ML-powered queries
+/javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
 
 # CodeQL tools and associated docs
 /docs/codeql/codeql-cli/ @github/codeql-cli-reviewers
@@ -41,8 +18,11 @@
 /docs/codeql/ql-language-reference/ @github/codeql-frontend-reviewers
 /docs/query-*-style-guide.md @github/codeql-analysis-reviewers
 
+# QL for QL reviewers
+/ql/ @github/codeql-ql-for-ql-reviewers
+
 # Bazel (excluding BUILD.bazel files)
-MODULE.bazel @github/codeql-ci-reviewers
+WORKSPACE.bazel @github/codeql-ci-reviewers
 .bazelversion @github/codeql-ci-reviewers
 .bazelrc @github/codeql-ci-reviewers
 **/*.bzl @github/codeql-ci-reviewers
@@ -53,15 +33,12 @@ MODULE.bazel @github/codeql-ci-reviewers
 
 # Workflows
 /.github/workflows/ @github/codeql-ci-reviewers
+/.github/workflows/atm-* @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/go-* @github/codeql-go
+/.github/workflows/js-ml-tests.yml @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
 /.github/workflows/ruby-* @github/codeql-ruby
-/.github/workflows/rust.yml @github/codeql-rust
 /.github/workflows/swift.yml @github/codeql-swift
 
 # Misc
 /misc/scripts/accept-expected-changes-from-ci.py @RasmusWL
-/misc/scripts/generate-code-scanning-query-list.py @RasmusWL
-
-# .devcontainer
-/.devcontainer/ @github/codeql-ci-reviewers

CONTRIBUTING.md

@@ -4,8 +4,6 @@ We welcome contributions to our CodeQL libraries and queries. Got an idea for a
 
 There is lots of useful documentation to help you write queries, ranging from information about query file structure to tutorials for specific target languages. For more information on the documentation available, see [CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/codeql-queries) on [codeql.github.com](https://codeql.github.com).
 
-Note that the CodeQL for Visual Studio Code documentation has been migrated to https://docs.github.com/en/code-security/codeql-for-vs-code/, but you can still contribute to it via a different repository. For more information, see [Contributing to GitHub Docs documentation](https://docs.github.com/en/contributing)."
-
 ## Change notes
 
 Any nontrivial user-visible change to a query pack or library pack should have a change note. For details on how to add a change note for your change, see [this guide](docs/change-notes.md).
@@ -45,7 +43,7 @@ If you have an idea for a query that you would like to share with other CodeQL u
 
 3. **Formatting**
 
-    - The queries and libraries must be autoformatted, for example using the "Format Document" command in [CodeQL for Visual Studio Code](https://docs.github.com/en/code-security/codeql-for-vs-code/).
+    - The queries and libraries must be autoformatted, for example using the "Format Document" command in [CodeQL for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-codeql-for-visual-studio-code).
 
     If you prefer, you can either:
     1. install the [pre-commit framework](https://pre-commit.com/) and install the configured hooks on this repo via `pre-commit install`, or

Cargo.toml

@@ -1,15 +0,0 @@
-# This is the shared workspace file for extractor using shared/tree-sitter/extractor
-[workspace]
-
-resolver = "2"
-members = [
-    "shared/tree-sitter-extractor",
-    "shared/yeast",
-    "shared/yeast-macros",
-    "ruby/extractor",
-    "unified/extractor",
-    "rust/extractor",
-    "rust/extractor/macros",
-    "rust/ast-generator",
-    "rust/autobuild",
-]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2006-2025 GitHub, Inc.
+Copyright (c) 2006-2020 GitHub, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

MODULE.bazel

@@ -1,294 +0,0 @@
-module(
-    name = "ql",
-    version = "0.0",
-    repo_name = "codeql",
-)
-
-# this points to our internal repository when `codeql` is checked out as a submodule thereof
-# when building things from `codeql` independently this is stubbed out in `.bazelrc`
-bazel_dep(name = "semmle_code", version = "0.0")
-local_path_override(
-    module_name = "semmle_code",
-    path = "..",
-)
-
-# see https://registry.bazel.build/ for a list of available packages
-
-bazel_dep(name = "platforms", version = "1.0.0")
-bazel_dep(name = "rules_cc", version = "0.2.17")
-bazel_dep(name = "rules_go", version = "0.60.0")
-bazel_dep(name = "rules_java", version = "9.6.1")
-bazel_dep(name = "rules_pkg", version = "1.2.0")
-bazel_dep(name = "rules_nodejs", version = "6.7.3")
-bazel_dep(name = "rules_python", version = "1.9.0")
-bazel_dep(name = "rules_shell", version = "0.7.1")
-bazel_dep(name = "bazel_skylib", version = "1.9.0")
-bazel_dep(name = "abseil-cpp", version = "20260107.1", repo_name = "absl")
-bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
-bazel_dep(name = "fmt", version = "12.1.0-codeql.1")
-bazel_dep(name = "rules_kotlin", version = "2.2.2-codeql.1")
-bazel_dep(name = "gazelle", version = "0.50.0")
-bazel_dep(name = "rules_dotnet", version = "0.21.5-codeql.1")
-bazel_dep(name = "googletest", version = "1.17.0.bcr.2")
-bazel_dep(name = "rules_rust", version = "0.69.0")
-bazel_dep(name = "zstd", version = "1.5.7.bcr.1")
-
-bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
-
-# Keep edition and version approximately in sync with internal repo.
-# the versions there are canonical, the versions here are used for CI in github/codeql, as well as for the vendoring of dependencies.
-RUST_EDITION = "2024"
-
-# run buildutils-internal/scripts/fill-rust-sha256s.py when updating (internal repo)
-# a nightly toolchain is required to enable experimental_use_cc_common_link, which we require internally
-# we prefer to run the same version as internally, even if experimental_use_cc_common_link is not really
-# required in this repo
-RUST_VERSION = "nightly/2026-01-22"
-
-rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
-rust.toolchain(
-    edition = RUST_EDITION,
-    # We need those extra target triples so that we can build universal binaries on macos
-    extra_target_triples = [
-        "x86_64-apple-darwin",
-        "aarch64-apple-darwin",
-    ],
-    # generated by buildutils-internal/scripts/fill-rust-sha256s.py (internal repo)
-    sha256s = {
-        "2026-01-22/rustc-nightly-x86_64-unknown-linux-gnu.tar.xz": "88db619323cc1321630d124efa51ed02fabc5e020f08cfa0eda2c0ac1afbe69a",
-        "2026-01-22/rustc-nightly-x86_64-apple-darwin.tar.xz": "08484da3fa38db56f93629aeabdc0ae9ff8ed9704c0792d35259cbc849b3f54c",
-        "2026-01-22/rustc-nightly-aarch64-apple-darwin.tar.xz": "a39c0b21b7058e364ea1bd43144e42e4bf1efade036b2e82455f2afce194ee81",
-        "2026-01-22/rustc-nightly-x86_64-pc-windows-msvc.tar.xz": "d00248ee9850dbb6932b2578e32ff74fc7c429854c1aa071066ca31b65385a3b",
-        "2026-01-22/clippy-nightly-x86_64-unknown-linux-gnu.tar.xz": "70656a0ce994ffff16d5a35a7b170a0acd41e9bb54a589c96ed45bf97b094a4d",
-        "2026-01-22/clippy-nightly-x86_64-apple-darwin.tar.xz": "fe242519fa961522734733009705aec3c2d9a20cc57291f2aa614e5e6262c88f",
-        "2026-01-22/clippy-nightly-aarch64-apple-darwin.tar.xz": "38bb226363ec97c9722edf966cd58774a683e19fd2ff2a6030094445d51e06f9",
-        "2026-01-22/clippy-nightly-x86_64-pc-windows-msvc.tar.xz": "6da9b4470beea67abfebf046f141eee0d2a8db7c7a9e4e2294478734fd477228",
-        "2026-01-22/cargo-nightly-x86_64-unknown-linux-gnu.tar.xz": "99004e9d10c43a01499642f53bb3184d41137a95d65bfb217098840a9e79e892",
-        "2026-01-22/cargo-nightly-x86_64-apple-darwin.tar.xz": "6e021394cf8d8400ac6cfdfcef24e4d74f988e91eb8028b36de3a64ce3502990",
-        "2026-01-22/cargo-nightly-aarch64-apple-darwin.tar.xz": "4b2494cb69ab64132cddbc411a38ea9f1105e54d6f986e43168d54f79510c673",
-        "2026-01-22/cargo-nightly-x86_64-pc-windows-msvc.tar.xz": "c36613cf57407212d10d37b76e49a60ff42336e953cdff9e177283f530a83fc1",
-        "2026-01-22/llvm-tools-nightly-x86_64-unknown-linux-gnu.tar.xz": "0b123c5027dbd833aae6845ffe9bd07d309bf798746a7176aadaea68fbcbd05d",
-        "2026-01-22/llvm-tools-nightly-x86_64-apple-darwin.tar.xz": "a47864491ad5619158c950ab7570fb6e487d5117338585c27334d45824b406d8",
-        "2026-01-22/llvm-tools-nightly-aarch64-apple-darwin.tar.xz": "db9bc826d6e2e7e914505d50157682e516ceb90357e83d77abddc32c2d962f41",
-        "2026-01-22/llvm-tools-nightly-x86_64-pc-windows-msvc.tar.xz": "ffaa406932b2fe62e01dad61cf4ed34860a5d2a6f9306ca340d79e630d930039",
-        "2026-01-22/rust-std-nightly-x86_64-unknown-linux-gnu.tar.xz": "e9c0d5e06e18a4b509391b3088f29293e310cdc8ccc865be8fa3f09733326925",
-        "2026-01-22/rust-std-nightly-x86_64-apple-darwin.tar.xz": "25d75995cee679a4828ca9fe48c5a31a67c3b0846018440ef912e5a6208f53f6",
-        "2026-01-22/rust-std-nightly-aarch64-apple-darwin.tar.xz": "e4132bf3f2eed4684c86756a02315bcf481c23e675e3e25630fc604c9cb4594c",
-        "2026-01-22/rust-std-nightly-x86_64-pc-windows-msvc.tar.xz": "961bb535ef95ae8a5fa4e224cb94aff190f155c45a9bcf7a53e184b024aa41b1",
-    },
-    versions = [RUST_VERSION],
-)
-use_repo(rust, "rust_toolchains")
-
-register_toolchains("@rust_toolchains//:all")
-
-# deps for python extractor
-# keep in sync by running `misc/bazel/3rdparty/update_cargo_deps.sh`
-py_deps = use_extension("//misc/bazel/3rdparty:py_deps_extension.bzl", "p")
-use_repo(
-    py_deps,
-    "vendor_py__anyhow-1.0.95",
-    "vendor_py__cc-1.2.14",
-    "vendor_py__clap-4.5.30",
-    "vendor_py__regex-1.11.1",
-    "vendor_py__tree-sitter-0.24.7",
-    "vendor_py__tree-sitter-graph-0.12.0",
-)
-
-# deps for ruby+rust
-# keep in sync by running `misc/bazel/3rdparty/update_cargo_deps.sh`
-tree_sitter_extractors_deps = use_extension("//misc/bazel/3rdparty:tree_sitter_extractors_extension.bzl", "r")
-use_repo(
-    tree_sitter_extractors_deps,
-    "vendor_ts__anyhow-1.0.100",
-    "vendor_ts__argfile-0.2.1",
-    "vendor_ts__chalk-ir-0.104.0",
-    "vendor_ts__chrono-0.4.42",
-    "vendor_ts__clap-4.5.48",
-    "vendor_ts__dunce-1.0.5",
-    "vendor_ts__either-1.15.0",
-    "vendor_ts__encoding-0.2.33",
-    "vendor_ts__figment-0.10.19",
-    "vendor_ts__flate2-1.1.2",
-    "vendor_ts__glob-0.3.3",
-    "vendor_ts__globset-0.4.16",
-    "vendor_ts__itertools-0.14.0",
-    "vendor_ts__lazy_static-1.5.0",
-    "vendor_ts__mustache-0.9.0",
-    "vendor_ts__num-traits-0.2.19",
-    "vendor_ts__num_cpus-1.17.0",
-    "vendor_ts__proc-macro2-1.0.101",
-    "vendor_ts__quote-1.0.41",
-    "vendor_ts__ra_ap_base_db-0.0.301",
-    "vendor_ts__ra_ap_cfg-0.0.301",
-    "vendor_ts__ra_ap_hir-0.0.301",
-    "vendor_ts__ra_ap_hir_def-0.0.301",
-    "vendor_ts__ra_ap_hir_expand-0.0.301",
-    "vendor_ts__ra_ap_hir_ty-0.0.301",
-    "vendor_ts__ra_ap_ide_db-0.0.301",
-    "vendor_ts__ra_ap_intern-0.0.301",
-    "vendor_ts__ra_ap_load-cargo-0.0.301",
-    "vendor_ts__ra_ap_parser-0.0.301",
-    "vendor_ts__ra_ap_paths-0.0.301",
-    "vendor_ts__ra_ap_project_model-0.0.301",
-    "vendor_ts__ra_ap_span-0.0.301",
-    "vendor_ts__ra_ap_stdx-0.0.301",
-    "vendor_ts__ra_ap_syntax-0.0.301",
-    "vendor_ts__ra_ap_vfs-0.0.301",
-    "vendor_ts__rand-0.9.2",
-    "vendor_ts__rayon-1.11.0",
-    "vendor_ts__regex-1.11.3",
-    "vendor_ts__serde-1.0.228",
-    "vendor_ts__serde_json-1.0.145",
-    "vendor_ts__serde_with-3.14.1",
-    "vendor_ts__serde_yaml-0.9.34-deprecated",
-    "vendor_ts__syn-2.0.106",
-    "vendor_ts__toml-0.9.7",
-    "vendor_ts__tracing-0.1.41",
-    "vendor_ts__tracing-flame-0.2.0",
-    "vendor_ts__tracing-subscriber-0.3.20",
-    "vendor_ts__tree-sitter-0.26.8",
-    "vendor_ts__tree-sitter-embedded-template-0.25.0",
-    "vendor_ts__tree-sitter-json-0.24.8",
-    "vendor_ts__tree-sitter-python-0.23.6",
-    "vendor_ts__tree-sitter-ql-0.23.1",
-    "vendor_ts__tree-sitter-ruby-0.23.1",
-    "vendor_ts__tree-sitter-swift-0.7.2",
-    "vendor_ts__triomphe-0.1.14",
-    "vendor_ts__ungrammar-1.16.1",
-    "vendor_ts__zstd-0.13.3",
-)
-
-http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-
-# rust-analyzer sources needed by the rust ast-generator (see `rust/ast-generator/README.md`)
-RUST_ANALYZER_SRC_TAG = "2025-01-07"
-
-http_archive(
-    name = "rust-analyzer-src",
-    build_file = "//rust/ast-generator:BUILD.rust-analyzer-src.bazel",
-    integrity = "sha256-eo8mIaUafZL8LOM65bDIIIXw1rNQ/P/x5RK/XUtgo5g=",
-    patch_args = ["-p1"],
-    patches = [
-        "//rust/ast-generator:patches/rust-analyzer.patch",
-    ],
-    strip_prefix = "rust-analyzer-%s" % RUST_ANALYZER_SRC_TAG,
-    url = "https://github.com/rust-lang/rust-analyzer/archive/refs/tags/%s.tar.gz" % RUST_ANALYZER_SRC_TAG,
-)
-
-dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet")
-dotnet.toolchain(dotnet_version = "10.0.100")
-use_repo(dotnet, "dotnet_toolchains")
-
-register_toolchains("@dotnet_toolchains//:all")
-
-csharp_main_extension = use_extension("//csharp:paket.main_extension.bzl", "main_extension")
-use_repo(csharp_main_extension, "paket.main")
-
-pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
-pip.parse(
-    hub_name = "codegen_deps",
-    python_version = "3.12",
-    requirements_lock = "//misc/codegen:requirements_lock.txt",
-)
-use_repo(pip, "codegen_deps")
-
-python = use_extension("@rules_python//python/extensions:python.bzl", "python")
-python.toolchain(
-    is_default = True,
-    python_version = "3.12",
-)
-use_repo(python, "python_3_12", "python_versions")
-
-register_toolchains("@python_versions//3.12:all")
-
-swift_deps = use_extension("//swift/third_party:load.bzl", "swift_deps")
-
-# following list can be kept in sync with `bazel mod tidy`
-use_repo(
-    swift_deps,
-    "binlog",
-    "picosha2",
-    "swift-prebuilt-linux",
-    "swift-prebuilt-linux-download-only",
-    "swift-prebuilt-macos",
-    "swift-prebuilt-macos-download-only",
-    "swift-resource-dir-linux",
-    "swift-resource-dir-macos",
-)
-
-node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node")
-node.toolchain(
-    name = "nodejs",
-    node_urls = [
-        "https://nodejs.org/dist/v{version}/{filename}",
-        "https://mirrors.dotsrc.org/nodejs/release/v{version}/{filename}",
-    ],
-    node_version = "18.15.0",
-)
-use_repo(node, "nodejs", "nodejs_toolchains")
-
-kotlin_extractor_deps = use_extension("//java/kotlin-extractor:deps.bzl", "kotlin_extractor_deps")
-
-# following list can be kept in sync by running `bazel mod tidy` in `codeql`
-use_repo(
-    kotlin_extractor_deps,
-    "codeql_kotlin_defaults",
-    "codeql_kotlin_embeddable",
-    "kotlin-compiler-1.8.0",
-    "kotlin-compiler-1.9.0-Beta",
-    "kotlin-compiler-1.9.20-Beta",
-    "kotlin-compiler-2.0.0-RC1",
-    "kotlin-compiler-2.0.20-Beta2",
-    "kotlin-compiler-2.1.0-Beta1",
-    "kotlin-compiler-2.1.20-Beta1",
-    "kotlin-compiler-2.2.0-Beta1",
-    "kotlin-compiler-2.2.20-Beta2",
-    "kotlin-compiler-2.3.0",
-    "kotlin-compiler-2.3.20",
-    "kotlin-compiler-embeddable-1.8.0",
-    "kotlin-compiler-embeddable-1.9.0-Beta",
-    "kotlin-compiler-embeddable-1.9.20-Beta",
-    "kotlin-compiler-embeddable-2.0.0-RC1",
-    "kotlin-compiler-embeddable-2.0.20-Beta2",
-    "kotlin-compiler-embeddable-2.1.0-Beta1",
-    "kotlin-compiler-embeddable-2.1.20-Beta1",
-    "kotlin-compiler-embeddable-2.2.0-Beta1",
-    "kotlin-compiler-embeddable-2.2.20-Beta2",
-    "kotlin-compiler-embeddable-2.3.0",
-    "kotlin-compiler-embeddable-2.3.20",
-    "kotlin-stdlib-1.8.0",
-    "kotlin-stdlib-1.9.0-Beta",
-    "kotlin-stdlib-1.9.20-Beta",
-    "kotlin-stdlib-2.0.0-RC1",
-    "kotlin-stdlib-2.0.20-Beta2",
-    "kotlin-stdlib-2.1.0-Beta1",
-    "kotlin-stdlib-2.1.20-Beta1",
-    "kotlin-stdlib-2.2.0-Beta1",
-    "kotlin-stdlib-2.2.20-Beta2",
-    "kotlin-stdlib-2.3.0",
-    "kotlin-stdlib-2.3.20",
-)
-
-go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
-go_sdk.download(version = "1.26.0")
-
-go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
-go_deps.from_file(go_mod = "//go/extractor:go.mod")
-use_repo(go_deps, "com_github_stretchr_testify", "org_golang_x_mod", "org_golang_x_tools")
-
-ripunzip_archive = use_repo_rule("//misc/ripunzip:ripunzip.bzl", "ripunzip_archive")
-
-# go to https://github.com/GoogleChrome/ripunzip/releases to find latest version and corresponding sha256s
-ripunzip_archive(
-    name = "ripunzip",
-    sha256_linux = "71482d7a7e4ea9176d5596161c49250c34b136b157c45f632b1111323fbfc0de",
-    sha256_macos_arm = "604194ab13f0aba3972995d995f11002b8fc285c8170401fcd46655065df20c9",
-    sha256_macos_intel = "65367b94fd579d93d46f2d2595cc4c9a60cfcf497e3c824f9d1a7b80fa8bd38a",
-    sha256_windows = "ac3874075def2b9e5074a3b5945005ab082cc6e689e1de658da8965bc23e643e",
-    version = "2.0.4",
-)
-
-register_toolchains(
-    "@nodejs_toolchains//:all",
-)

README.md

@@ -4,7 +4,7 @@ This open source repository contains the standard CodeQL libraries and queries t
 
 ## How do I learn CodeQL and run queries?
 
-There is extensive documentation about the [CodeQL language](https://codeql.github.com/docs/), writing CodeQL using the [CodeQL extension for Visual Studio Code](https://docs.github.com/en/code-security/codeql-for-vs-code/) and using the [CodeQL CLI](https://docs.github.com/en/code-security/codeql-cli).
+There is [extensive documentation](https://codeql.github.com/docs/) on getting started with writing CodeQL using the [CodeQL extension for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/) and the [CodeQL CLI](https://codeql.github.com/docs/codeql-cli/).
 
 ## Contributing
 

WORKSPACE.bazel

@@ -0,0 +1,12 @@
+# Please notice that any bazel targets and definitions in this repository are currently experimental
+# and for internal use only.
+
+workspace(name = "codeql")
+
+load("//misc/bazel:workspace.bzl", "codeql_workspace")
+
+codeql_workspace()
+
+load("//misc/bazel:workspace_deps.bzl", "codeql_workspace_deps")
+
+codeql_workspace_deps()

actions/BUILD.bazel

@@ -1,9 +0,0 @@
-load("//misc/bazel:pkg.bzl", "codeql_pack")
-
-package(default_visibility = ["//visibility:public"])
-
-codeql_pack(
-    name = "actions",
-    srcs = ["//actions/extractor"],
-    experimental = True,
-)

actions/extractor/BUILD.bazel

@@ -1,12 +0,0 @@
-load("//misc/bazel:pkg.bzl", "codeql_pkg_files", "strip_prefix")
-
-codeql_pkg_files(
-    name = "extractor",
-    srcs = [
-        "codeql-extractor.yml",
-        "//:LICENSE",
-    ],
-    exes = glob(["tools/**"]),
-    strip_prefix = strip_prefix.from_pkg(),
-    visibility = ["//actions:__pkg__"],
-)

actions/extractor/codeql-extractor.yml

@@ -1,47 +0,0 @@
-name: "actions"
-display_name: "GitHub Actions"
-version: 0.0.1
-column_kind: "utf16"
-unicode_newlines: true
-build_modes:
-  - none
-default_queries:
-  - codeql/actions-queries
-# Actions workflows are not reported separately by the GitHub API, so we can't
-# associate them with a specific language.
-github_api_languages: []
-scc_languages:
-  - YAML
-file_types:
-  - name: workflow
-    display_name: GitHub Actions workflow files
-    extensions:
-      - .yml
-      - .yaml
-forwarded_extractor_name: javascript
-options:
-  trap:
-    title: TRAP options
-    description: Options about how the extractor handles TRAP files
-    type: object
-    visibility: 3
-    properties:
-      cache:
-        title: TRAP cache options
-        description: Options about how the extractor handles its TRAP cache
-        type: object
-        properties:
-          dir:
-            title: TRAP cache directory
-            description: The directory of the TRAP cache to use
-            type: string
-          bound:
-            title: TRAP cache bound
-            description: A soft limit (in MB) on the size of the TRAP cache
-            type: string
-            pattern: "[0-9]+"
-          write:
-            title: TRAP cache writeable
-            description: Whether to write to the TRAP cache as well as reading it
-            type: string
-            pattern: "(true|TRUE|false|FALSE)"

actions/extractor/tools/autobuild-impl.ps1

@@ -1,53 +0,0 @@
-# Note: We're adding the `reusable_workflows` subdirectories to proactively
-# record workflows that were called cross-repo, check them out locally,
-# and enable an interprocedural analysis across the workflow files.
-# These workflows follow the convention `.github/reusable_workflows/<nwo>/*.ya?ml`
-$DefaultPathFilters = @(
-    'exclude:**/*',
-    'include:.github/workflows/*.yml',
-    'include:.github/workflows/*.yaml',
-    'include:.github/reusable_workflows/**/*.yml',
-    'include:.github/reusable_workflows/**/*.yaml',
-    'include:**/action.yml',
-    'include:**/action.yaml'
-)
-
-if ($null -ne $env:LGTM_INDEX_FILTERS) {
-    Write-Output 'LGTM_INDEX_FILTERS set. Using the default filters together with the user-provided filters, and passing through to the JavaScript extractor.'
-    # Begin with the default path inclusions only,
-    # followed by the user-provided filters.
-    # If the user provided `paths`, those patterns override the default inclusions
-    # (because `LGTM_INDEX_FILTERS` will begin with `exclude:**/*`).
-    # If the user provided `paths-ignore`, those patterns are excluded.
-    $PathFilters = ($DefaultPathFilters -join "`n") + "`n" + $env:LGTM_INDEX_FILTERS
-    $env:LGTM_INDEX_FILTERS = $PathFilters
-} else {
-    Write-Output 'LGTM_INDEX_FILTERS not set. Using the default filters, and passing through to the JavaScript extractor.'
-    $env:LGTM_INDEX_FILTERS = $DefaultPathFilters -join "`n"
-}
-
-# Find the JavaScript extractor directory via `codeql resolve extractor`.
-$CodeQL = Join-Path $env:CODEQL_DIST 'codeql.exe'
-$env:CODEQL_EXTRACTOR_JAVASCRIPT_ROOT = &"$CodeQL" resolve extractor --language javascript
-if ($LASTEXITCODE -ne 0) {
-    throw 'Failed to resolve JavaScript extractor.'
-}
-
-Write-Output "Found JavaScript extractor at '${env:CODEQL_EXTRACTOR_JAVASCRIPT_ROOT}'."
-
-# Run the JavaScript autobuilder.
-$JavaScriptAutoBuild = Join-Path $env:CODEQL_EXTRACTOR_JAVASCRIPT_ROOT 'tools\autobuild.cmd'
-Write-Output "Running JavaScript autobuilder at '${JavaScriptAutoBuild}'."
-
-# Copy the values of the Actions extractor environment variables to the JavaScript extractor environment variables.
-$env:CODEQL_EXTRACTOR_JAVASCRIPT_DIAGNOSTIC_DIR = $env:CODEQL_EXTRACTOR_ACTIONS_DIAGNOSTIC_DIR
-$env:CODEQL_EXTRACTOR_JAVASCRIPT_LOG_DIR = $env:CODEQL_EXTRACTOR_ACTIONS_LOG_DIR
-$env:CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR = $env:CODEQL_EXTRACTOR_ACTIONS_SCRATCH_DIR
-$env:CODEQL_EXTRACTOR_JAVASCRIPT_SOURCE_ARCHIVE_DIR = $env:CODEQL_EXTRACTOR_ACTIONS_SOURCE_ARCHIVE_DIR
-$env:CODEQL_EXTRACTOR_JAVASCRIPT_TRAP_DIR = $env:CODEQL_EXTRACTOR_ACTIONS_TRAP_DIR
-$env:CODEQL_EXTRACTOR_JAVASCRIPT_WIP_DATABASE = $env:CODEQL_EXTRACTOR_ACTIONS_WIP_DATABASE
-
-&"$JavaScriptAutoBuild"
-if ($LASTEXITCODE -ne 0) {
-    throw "JavaScript autobuilder failed."
-}

actions/extractor/tools/autobuild.cmd

@@ -1,4 +0,0 @@
-@echo off
-rem All of the work is done in the PowerShell script
-echo "Running PowerShell script at '%~dp0autobuild-impl.ps1'"
-powershell.exe -File "%~dp0autobuild-impl.ps1"

actions/extractor/tools/autobuild.sh

@@ -1,57 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-# Note: We're adding the `reusable_workflows` subdirectories to proactively
-# record workflows that were called cross-repo, check them out locally,
-# and enable an interprocedural analysis across the workflow files.
-# These workflows follow the convention `.github/reusable_workflows/<nwo>/*.ya?ml`
-DEFAULT_PATH_FILTERS=$(cat << END
-exclude:**/*
-include:.github/workflows/*.yml
-include:.github/workflows/*.yaml
-include:.github/reusable_workflows/**/*.yml
-include:.github/reusable_workflows/**/*.yaml
-include:**/action.yml
-include:**/action.yaml
-END
-)
-
-if [ -n "${LGTM_INDEX_FILTERS:-}" ]; then
-    echo "LGTM_INDEX_FILTERS set. Using the default filters together with the user-provided filters, and passing through to the JavaScript extractor."
-    # Begin with the default path inclusions only,
-    # followed by the user-provided filters.
-    # If the user provided `paths`, those patterns override the default inclusions
-    # (because `LGTM_INDEX_FILTERS` will begin with `exclude:**/*`).
-    # If the user provided `paths-ignore`, those patterns are excluded.
-    PATH_FILTERS="$(cat << END
-${DEFAULT_PATH_FILTERS}
-${LGTM_INDEX_FILTERS}
-END
-)"
-    LGTM_INDEX_FILTERS="${PATH_FILTERS}"
-    export LGTM_INDEX_FILTERS
-else
-    echo "LGTM_INDEX_FILTERS not set. Using the default filters, and passing through to the JavaScript extractor."
-    LGTM_INDEX_FILTERS="${DEFAULT_PATH_FILTERS}"
-    export LGTM_INDEX_FILTERS
-fi
-
-# Find the JavaScript extractor directory via `codeql resolve extractor`.
-CODEQL_EXTRACTOR_JAVASCRIPT_ROOT="$("${CODEQL_DIST}/codeql" resolve extractor --language javascript)"
-export CODEQL_EXTRACTOR_JAVASCRIPT_ROOT
-
-echo "Found JavaScript extractor at '${CODEQL_EXTRACTOR_JAVASCRIPT_ROOT}'."
-
-# Run the JavaScript autobuilder
-JAVASCRIPT_AUTO_BUILD="${CODEQL_EXTRACTOR_JAVASCRIPT_ROOT}/tools/autobuild.sh"
-echo "Running JavaScript autobuilder at '${JAVASCRIPT_AUTO_BUILD}'."
-
-# Copy the values of the Actions extractor environment variables to the JavaScript extractor environment variables.
-env CODEQL_EXTRACTOR_JAVASCRIPT_DIAGNOSTIC_DIR="${CODEQL_EXTRACTOR_ACTIONS_DIAGNOSTIC_DIR}" \
-    CODEQL_EXTRACTOR_JAVASCRIPT_LOG_DIR="${CODEQL_EXTRACTOR_ACTIONS_LOG_DIR}" \
-    CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR="${CODEQL_EXTRACTOR_ACTIONS_SCRATCH_DIR}" \
-    CODEQL_EXTRACTOR_JAVASCRIPT_SOURCE_ARCHIVE_DIR="${CODEQL_EXTRACTOR_ACTIONS_SOURCE_ARCHIVE_DIR}" \
-    CODEQL_EXTRACTOR_JAVASCRIPT_TRAP_DIR="${CODEQL_EXTRACTOR_ACTIONS_TRAP_DIR}" \
-    CODEQL_EXTRACTOR_JAVASCRIPT_WIP_DATABASE="${CODEQL_EXTRACTOR_ACTIONS_WIP_DATABASE}" \
-    "${JAVASCRIPT_AUTO_BUILD}"

actions/extractor/tools/baseline-config.json

@@ -1,10 +0,0 @@
-{
-    "paths": [
-        ".github/workflows/*.yml",
-        ".github/workflows/*.yaml",
-        ".github/reusable_workflows/**/*.yml",
-        ".github/reusable_workflows/**/*.yaml",
-        "**/action.yml",
-        "**/action.yaml"
-    ]
-}

actions/extractor/tools/configure-baseline.cmd

@@ -1,2 +0,0 @@
-@echo off
-type "%CODEQL_EXTRACTOR_ACTIONS_ROOT%\tools\baseline-config.json"

actions/extractor/tools/configure-baseline.sh

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-cat "$CODEQL_EXTRACTOR_ACTIONS_ROOT/tools/baseline-config.json"

actions/ql/examples/codeql-pack.lock.yml

@@ -1,4 +0,0 @@
----
-lockVersion: 1.0.0
-dependencies: {}
-compiled: false

actions/ql/examples/qlpack.yml

@@ -1,7 +0,0 @@
-name: codeql/actions-examples
-groups:
-  - actions
-  - examples
-dependencies:
-  codeql/actions-all: ${workspace}
-warnOnImplicitThis: true

actions/ql/examples/snippets/uses_pinned_sha.ql

@@ -1,12 +0,0 @@
-/**
- * @name Uses step with pinned SHA
- * @description Finds 'uses' steps where the version is a pinned SHA.
- * @id actions/examples/uses-pinned-sha
- * @tags example
- */
-
-import actions
-
-from UsesStep uses
-where uses.getVersion().regexpMatch("^[A-Fa-f0-9]{40}$")
-select uses, "This 'uses' step has a pinned SHA version."

actions/ql/extensions/immutable-actions-list/ext/immutable_actions.yml

@@ -1,28 +0,0 @@
-extensions:
-  - addsTo:
-      pack: codeql/actions-all
-      extensible: immutableActionsDataModel
-    data:
-      - ["actions/checkout"]
-      - ["actions/cache"]
-      - ["actions/setup-node"]
-      - ["actions/upload-artifact"]
-      - ["actions/setup-python"]
-      - ["actions/download-artifact"]
-      - ["actions/github-script"]
-      - ["actions/setup-java"]
-      - ["actions/setup-go"]
-      - ["actions/upload-pages-artifact"]
-      - ["actions/deploy-pages"]
-      - ["actions/setup-dotnet"]
-      - ["actions/stale"]
-      - ["actions/labeler"]
-      - ["actions/create-github-app-token"]
-      - ["actions/configure-pages"]
-      - ["github/codeql-action/analyze"]
-      - ["github/codeql-action/autobuild"]
-      - ["github/codeql-action/init"]
-      - ["github/codeql-action/resolve-environment"]
-      - ["github/codeql-action/start-proxy"]
-      - ["github/codeql-action/upload-sarif"]
-      - ["octokit/request-action"]

actions/ql/extensions/immutable-actions-list/qlpack.yml

@@ -1,14 +0,0 @@
-# Model pack containing the list of known immutable actions. The Immutable Actions feature is not
-# yet released, so this pack will only be used within GitHub. Once the feature is available to
-# customers, we will move the contents of this pack back into the standard library pack.
-name: codeql/immutable-actions-list
-version: 0.0.1-dev
-library: true
-warnOnImplicitThis: true
-extensionTargets:
-  # We expect to need this model pack even after GA of Actions analysis, so make it compatible with
-  # all future prereleases plus 1.x.x. We should be able to remove this back before we need to
-  # bump the major version to 2.
-  codeql/actions-all: ">=0.4.3 <2.0.0"
-dataExtensions:
-- ext/**/*.yml

actions/ql/integration-tests/filters-default/actions.ql

@@ -1,5 +0,0 @@
-import actions
-
-from AstNode n
-where n instanceof Workflow or n instanceof CompositeAction
-select n

actions/ql/integration-tests/filters/actions.default-filters.expected

@@ -1,6 +0,0 @@
-| src/.github/action.yaml:1:1:11:32 | name: ' ... action' |
-| src/.github/actions/action-name/action.yml:1:1:11:32 | name: ' ... action' |
-| src/.github/workflows/workflow.yml:1:1:12:33 | name: A workflow |
-| src/action.yml:1:1:11:32 | name: ' ... action' |
-| src/excluded/action.yml:1:1:11:32 | name: ' ... action' |
-| src/included/action.yml:1:1:11:32 | name: ' ... action' |

actions/ql/integration-tests/filters/actions.paths-and-paths-ignore.expected

@@ -1,2 +0,0 @@
-| src/included/action.yml:1:1:11:32 | name: ' ... action' |
-| src/included/unreachable-workflow.yml:1:1:12:33 | name: A ... orkflow |

actions/ql/integration-tests/filters/actions.paths-ignore-only.expected

@@ -1,5 +0,0 @@
-| src/.github/action.yaml:1:1:11:32 | name: ' ... action' |
-| src/.github/actions/action-name/action.yml:1:1:11:32 | name: ' ... action' |
-| src/.github/workflows/workflow.yml:1:1:12:33 | name: A workflow |
-| src/action.yml:1:1:11:32 | name: ' ... action' |
-| src/included/action.yml:1:1:11:32 | name: ' ... action' |

actions/ql/integration-tests/filters/actions.paths-only.expected

@@ -1,2 +0,0 @@
-| src/included/action.yml:1:1:11:32 | name: ' ... action' |
-| src/included/unreachable-workflow.yml:1:1:12:33 | name: A ... orkflow |

actions/ql/integration-tests/filters/actions.ql

@@ -1,5 +0,0 @@
-import actions
-
-from AstNode n
-where n instanceof Workflow or n instanceof CompositeAction
-select n

actions/ql/integration-tests/filters/codeql-config.paths-and-paths-ignore.yml

@@ -1,4 +0,0 @@
-paths:
-  - 'included'
-paths-ignore:
-  - 'excluded'

actions/ql/integration-tests/filters/codeql-config.paths-ignore-only.yml

@@ -1,2 +0,0 @@
-paths-ignore:
-  - 'excluded'

actions/ql/integration-tests/filters/codeql-config.paths-only.yml

@@ -1,2 +0,0 @@
-paths:
-  - 'included'

actions/ql/integration-tests/filters/source_archive.default-filters.expected

@@ -1,6 +0,0 @@
-src/.github/action.yaml
-src/.github/actions/action-name/action.yml
-src/.github/workflows/workflow.yml
-src/action.yml
-src/excluded/action.yml
-src/included/action.yml

actions/ql/integration-tests/filters/source_archive.paths-and-paths-ignore.expected

@@ -1,3 +0,0 @@
-src/included/action.yml
-src/included/not-an-action.yml
-src/included/unreachable-workflow.yml

actions/ql/integration-tests/filters/source_archive.paths-ignore-only.expected

@@ -1,5 +0,0 @@
-src/.github/action.yaml
-src/.github/actions/action-name/action.yml
-src/.github/workflows/workflow.yml
-src/action.yml
-src/included/action.yml

actions/ql/integration-tests/filters/source_archive.paths-only.expected

@@ -1,3 +0,0 @@
-src/included/action.yml
-src/included/not-an-action.yml
-src/included/unreachable-workflow.yml

actions/ql/integration-tests/filters/src/.github/action.yaml

@@ -1,11 +0,0 @@
-name: 'A composite action'
-description: 'Do something'
-runs:
-  using: "composite"
-  steps:
-    - name: Print
-      run: echo "Hello world"
-      shell: bash
-
-    - name: Checkout
-      uses: actions/checkout@v4