set CompileForOverlayEval in java pack

Java: disable failing maven fetches expectations for now

.github/copilot-instructions.md

@@ -0,0 +1,4 @@
+When reviewing code:
+* do not review changes in files with `.expected` extension (they are automatically ensured to be correct).
+* in `.ql` and `.qll` files, do not try to review the code itself as you don't understand the programming language
+  well enough to make comments in these languages. You can still check for typos or comment improvements.

.github/workflows/build-ripunzip.yml

@@ -6,18 +6,18 @@ on:
       ripunzip-version:
         description: "what reference to checktout from google/runzip"
         required: false
-        default: v1.2.1
+        default: v2.0.2
       openssl-version:
         description: "what reference to checkout from openssl/openssl for Linux"
         required: false
-        default: openssl-3.3.0
+        default: openssl-3.5.0
 
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, macos-13, windows-2019]
+        os: [ubuntu-22.04, macos-13, windows-2022]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4

.github/workflows/check-change-note.yml

@@ -16,7 +16,6 @@ on:
       - "shared/**/*.qll"
       - "!**/experimental/**"
       - "!ql/**"
-      - "!rust/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:

.github/workflows/check-overlay-annotations.yml

@@ -0,0 +1,23 @@
+name: Check overlay annotations
+
+on:
+  push:
+    branches:
+      - main
+      - 'rc/*'
+  pull_request:
+    branches:
+      - main
+      - 'rc/*'
+
+permissions:
+  contents: read
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check overlay annotations
+        run: python config/add-overlay-annotations.py --check java
+

.github/workflows/codegen.yml

@@ -1,34 +0,0 @@
-name: Codegen
-
-on:
-  pull_request:
-    paths:
-      - "misc/bazel/**"
-      - "misc/codegen/**"
-      - "*.bazel*"
-      - .github/workflows/codegen.yml
-      - .pre-commit-config.yaml
-    branches:
-      - main
-      - rc/*
-      - codeql-cli-*
-
-permissions:
-  contents: read
-
-jobs:
-  codegen:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version-file: 'misc/codegen/.python-version'
-      - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
-        name: Check that python code is properly formatted
-        with:
-          extra_args: autopep8 --all-files
-      - name: Run codegen tests
-        shell: bash
-        run: |
-          bazel test //misc/codegen/...

.github/workflows/csharp-qltest.yml

@@ -36,7 +36,7 @@ jobs:
   unit-tests:
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-2019]
+        os: [ubuntu-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
@@ -66,6 +66,6 @@ jobs:
           # Update existing stubs in the repo with the freshly generated ones
           mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/
           git status
-          codeql test run --threads=0 --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
+          codeql test run --threads=0 --search-path "${{ github.workspace }}" --check-databases --check-diff-informed --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/python-tooling.yml

@@ -0,0 +1,35 @@
+name: Python tooling
+
+on:
+  pull_request:
+    paths:
+      - "misc/bazel/**"
+      - "misc/codegen/**"
+      - "misc/scripts/models-as-data/bulk_generate_mad.py"
+      - "*.bazel*"
+      - .github/workflows/codegen.yml
+      - .pre-commit-config.yaml
+    branches:
+      - main
+      - rc/*
+      - codeql-cli-*
+
+permissions:
+  contents: read
+
+jobs:
+  check-python-tooling:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
+        name: Check that python code is properly formatted
+        with:
+          extra_args: black --all-files
+      - name: Run codegen tests
+        shell: bash
+        run: |
+          bazel test //misc/codegen/...

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -53,7 +53,7 @@ jobs:
       - name: Create database
         run: |
           "${CODEQL}" database create \
-          --search-path "${{ github.workspace }}"
+          --search-path "${{ github.workspace }}" \
           --threads 4 \
             --language ql --source-root "${{ github.workspace }}/repo" \
             "${{ runner.temp }}/database"

.github/workflows/ruby-qltest-rtjo.yml

@@ -35,6 +35,6 @@ jobs:
           key: ruby-qltest
       - name: Run QL tests
         run: |
-          codeql test run --dynamic-join-order-mode=all --threads=0 --ram 50000 --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --dynamic-join-order-mode=all --threads=0 --ram 50000 --search-path "${{ github.workspace }}" --check-databases --check-diff-informed --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/ruby-qltest.yml

@@ -68,6 +68,6 @@ jobs:
           key: ruby-qltest
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}" --check-databases --check-diff-informed --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/validate-change-notes.yml

@@ -31,4 +31,4 @@ jobs:
       - name: Fail if there are any errors with existing change notes
 
         run: |
-          codeql pack release --groups cpp,csharp,java,javascript,python,ruby,-examples,-test,-experimental
+          codeql pack release --groups actions,cpp,csharp,go,java,javascript,python,ruby,shared,swift -examples,-test,-experimental

.pre-commit-config.yaml

@@ -1,5 +1,7 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
+default_language_version:
+  python: python3.12
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.2.0
@@ -7,18 +9,18 @@ repos:
       - id: trailing-whitespace
         exclude: /test/.*$(?<!\.qlref)|.*\.patch$|.*\.qll?$
       - id: end-of-file-fixer
-        exclude: /test/.*$(?<!\.qlref)|.*\.patch$|.*\.qll?$
+        exclude: Cargo.lock$|/test/.*$(?<!\.qlref)|.*\.patch$|.*\.qll?$
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: v17.0.6
     hooks:
       - id: clang-format
 
-  - repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v2.0.4
+  - repo: https://github.com/psf/black
+    rev: 25.1.0
     hooks:
-      - id: autopep8
-        files: ^misc/codegen/.*\.py
+      - id: black
+        files: ^(misc/codegen/.*|misc/scripts/models-as-data/.*)\.py$
 
   - repo: local
     hooks:

CODEOWNERS

@@ -16,7 +16,7 @@
 /java/ql/test-kotlin2/ @github/codeql-kotlin
 
 # Experimental CodeQL cryptography
-**/experimental/quantum/ @github/ps-codeql
+**/experimental/**/quantum/ @github/ps-codeql
 /shared/quantum/ @github/ps-codeql
 
 # CodeQL tools and associated docs

Cargo.toml

@@ -11,8 +11,3 @@ members = [
     "rust/autobuild",
 ]
 exclude = ["mad-generation-build"]
-
-[patch.crates-io]
-# patch for build script bug preventing bazel build
-# see https://github.com/rust-lang/rustc_apfloat/pull/17
-rustc_apfloat = { git = "https://github.com/redsun82/rustc_apfloat.git", rev = "32968f16ef1b082243f9bf43a3fbd65c381b3e27" }

MODULE.bazel

@@ -37,7 +37,7 @@ bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True
 # the versions there are canonical, the versions here are used for CI in github/codeql, as well as for the vendoring of dependencies.
 RUST_EDITION = "2024"
 
-RUST_VERSION = "1.85.0"
+RUST_VERSION = "1.86.0"
 
 rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
 rust.toolchain(
@@ -71,11 +71,11 @@ use_repo(
 tree_sitter_extractors_deps = use_extension("//misc/bazel/3rdparty:tree_sitter_extractors_extension.bzl", "r")
 use_repo(
     tree_sitter_extractors_deps,
-    "vendor_ts__anyhow-1.0.97",
+    "vendor_ts__anyhow-1.0.98",
     "vendor_ts__argfile-0.2.1",
-    "vendor_ts__chalk-ir-0.100.0",
-    "vendor_ts__chrono-0.4.40",
-    "vendor_ts__clap-4.5.35",
+    "vendor_ts__chalk-ir-0.103.0",
+    "vendor_ts__chrono-0.4.41",
+    "vendor_ts__clap-4.5.40",
     "vendor_ts__dunce-1.0.5",
     "vendor_ts__either-1.15.0",
     "vendor_ts__encoding-0.2.33",
@@ -87,33 +87,33 @@ use_repo(
     "vendor_ts__lazy_static-1.5.0",
     "vendor_ts__mustache-0.9.0",
     "vendor_ts__num-traits-0.2.19",
-    "vendor_ts__num_cpus-1.16.0",
-    "vendor_ts__proc-macro2-1.0.94",
+    "vendor_ts__num_cpus-1.17.0",
+    "vendor_ts__proc-macro2-1.0.95",
     "vendor_ts__quote-1.0.40",
-    "vendor_ts__ra_ap_base_db-0.0.273",
-    "vendor_ts__ra_ap_cfg-0.0.273",
-    "vendor_ts__ra_ap_hir-0.0.273",
-    "vendor_ts__ra_ap_hir_def-0.0.273",
-    "vendor_ts__ra_ap_hir_expand-0.0.273",
-    "vendor_ts__ra_ap_hir_ty-0.0.273",
-    "vendor_ts__ra_ap_ide_db-0.0.273",
-    "vendor_ts__ra_ap_intern-0.0.273",
-    "vendor_ts__ra_ap_load-cargo-0.0.273",
-    "vendor_ts__ra_ap_parser-0.0.273",
-    "vendor_ts__ra_ap_paths-0.0.273",
-    "vendor_ts__ra_ap_project_model-0.0.273",
-    "vendor_ts__ra_ap_span-0.0.273",
-    "vendor_ts__ra_ap_stdx-0.0.273",
-    "vendor_ts__ra_ap_syntax-0.0.273",
-    "vendor_ts__ra_ap_vfs-0.0.273",
-    "vendor_ts__rand-0.9.0",
+    "vendor_ts__ra_ap_base_db-0.0.288",
+    "vendor_ts__ra_ap_cfg-0.0.288",
+    "vendor_ts__ra_ap_hir-0.0.288",
+    "vendor_ts__ra_ap_hir_def-0.0.288",
+    "vendor_ts__ra_ap_hir_expand-0.0.288",
+    "vendor_ts__ra_ap_hir_ty-0.0.288",
+    "vendor_ts__ra_ap_ide_db-0.0.288",
+    "vendor_ts__ra_ap_intern-0.0.288",
+    "vendor_ts__ra_ap_load-cargo-0.0.288",
+    "vendor_ts__ra_ap_parser-0.0.288",
+    "vendor_ts__ra_ap_paths-0.0.288",
+    "vendor_ts__ra_ap_project_model-0.0.288",
+    "vendor_ts__ra_ap_span-0.0.288",
+    "vendor_ts__ra_ap_stdx-0.0.288",
+    "vendor_ts__ra_ap_syntax-0.0.288",
+    "vendor_ts__ra_ap_vfs-0.0.288",
+    "vendor_ts__rand-0.9.1",
     "vendor_ts__rayon-1.10.0",
     "vendor_ts__regex-1.11.1",
     "vendor_ts__serde-1.0.219",
     "vendor_ts__serde_json-1.0.140",
-    "vendor_ts__serde_with-3.12.0",
-    "vendor_ts__syn-2.0.100",
-    "vendor_ts__toml-0.8.20",
+    "vendor_ts__serde_with-3.13.0",
+    "vendor_ts__syn-2.0.103",
+    "vendor_ts__toml-0.8.23",
     "vendor_ts__tracing-0.1.41",
     "vendor_ts__tracing-flame-0.2.0",
     "vendor_ts__tracing-subscriber-0.3.19",
@@ -124,6 +124,7 @@ use_repo(
     "vendor_ts__tree-sitter-ruby-0.23.1",
     "vendor_ts__triomphe-0.1.14",
     "vendor_ts__ungrammar-1.16.1",
+    "vendor_ts__zstd-0.13.3",
 )
 
 http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
@@ -238,24 +239,24 @@ go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
 go_deps.from_file(go_mod = "//go/extractor:go.mod")
 use_repo(go_deps, "org_golang_x_mod", "org_golang_x_tools")
 
-lfs_files = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_files")
+lfs_archive = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_archive")
 
-lfs_files(
+lfs_archive(
     name = "ripunzip-linux",
-    srcs = ["//misc/ripunzip:ripunzip-linux"],
-    executable = True,
+    src = "//misc/ripunzip:ripunzip-Linux.zip",
+    build_file = "//misc/ripunzip:BUILD.ripunzip.bazel",
 )
 
-lfs_files(
+lfs_archive(
     name = "ripunzip-windows",
-    srcs = ["//misc/ripunzip:ripunzip-windows.exe"],
-    executable = True,
+    src = "//misc/ripunzip:ripunzip-Windows.zip",
+    build_file = "//misc/ripunzip:BUILD.ripunzip.bazel",
 )
 
-lfs_files(
+lfs_archive(
     name = "ripunzip-macos",
-    srcs = ["//misc/ripunzip:ripunzip-macos"],
-    executable = True,
+    src = "//misc/ripunzip:ripunzip-macOS.zip",
+    build_file = "//misc/ripunzip:BUILD.ripunzip.bazel",
 )
 
 register_toolchains(

actions/ql/integration-tests/query-suite/actions-code-quality-extended.qls.expected

@@ -0,0 +1 @@
+

actions/ql/integration-tests/query-suite/test.py

@@ -2,7 +2,7 @@ import runs_on
 import pytest
 from query_suites import *
 
-well_known_query_suites = ['actions-code-quality.qls', 'actions-security-and-quality.qls', 'actions-security-extended.qls', 'actions-code-scanning.qls']
+well_known_query_suites = ['actions-code-quality.qls', 'actions-code-quality-extended.qls', 'actions-security-and-quality.qls', 'actions-security-extended.qls', 'actions-code-scanning.qls']
 
 @runs_on.posix
 @pytest.mark.parametrize("query_suite", well_known_query_suites)

actions/ql/lib/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.4.12
+
+### Minor Analysis Improvements
+
+* Fixed performance issues in the parsing of Bash scripts in workflow files,
+  which led to out-of-disk errors when analysing certain workflow files with
+  complex interpolations of shell commands or quoted strings.
+
+## 0.4.11
+
+No user-facing changes.
+
 ## 0.4.10
 
 No user-facing changes.

actions/ql/lib/change-notes/released/0.4.11.md

@@ -0,0 +1,3 @@
+## 0.4.11
+
+No user-facing changes.

actions/ql/lib/change-notes/released/0.4.12.md

@@ -0,0 +1,7 @@
+## 0.4.12
+
+### Minor Analysis Improvements
+
+* Fixed performance issues in the parsing of Bash scripts in workflow files,
+  which led to out-of-disk errors when analysing certain workflow files with
+  complex interpolations of shell commands or quoted strings.

actions/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.4.10
+lastReleaseVersion: 0.4.12

actions/ql/lib/codeql/actions/Ast.qll

@@ -50,8 +50,8 @@ class Expression extends AstNode instanceof ExpressionImpl {
   string getNormalizedExpression() { result = normalizeExpr(expression) }
 }
 
-/** A common class for `env` in workflow, job or step. */
-abstract class Env extends AstNode instanceof EnvImpl {
+/** An `env` in workflow, job or step. */
+class Env extends AstNode instanceof EnvImpl {
   /** Gets an environment variable value given its name. */
   ScalarValueImpl getEnvVarValue(string name) { result = super.getEnvVarValue(name) }
 

actions/ql/lib/codeql/actions/Bash.qll

@@ -8,35 +8,64 @@ class BashShellScript extends ShellScript {
     )
   }
 
-  private string lineProducer(int i) {
-    result = this.getRawScript().regexpReplaceAll("\\\\\\s*\n", "").splitAt("\n", i)
+  /**
+   * Gets the line at 0-based index `lineIndex` within this shell script,
+   * assuming newlines as separators.
+   */
+  private string lineProducer(int lineIndex) {
+    result = this.getRawScript().regexpReplaceAll("\\\\\\s*\n", "").splitAt("\n", lineIndex)
   }
 
-  private predicate cmdSubstitutionReplacement(string cmdSubs, string id, int k) {
-    exists(string line | line = this.lineProducer(k) |
-      exists(int i, int j |
-        cmdSubs =
-          // $() cmd substitution
-          line.regexpFind("\\$\\((?:[^()]+|\\((?:[^()]+|\\([^()]*\\))*\\))*\\)", i, j)
-              .regexpReplaceAll("^\\$\\(", "")
-              .regexpReplaceAll("\\)$", "") and
-        id = "cmdsubs:" + k + ":" + i + ":" + j
-      )
-      or
-      exists(int i, int j |
-        // `...` cmd substitution
-        cmdSubs =
-          line.regexpFind("\\`[^\\`]+\\`", i, j)
-              .regexpReplaceAll("^\\`", "")
-              .regexpReplaceAll("\\`$", "") and
-        id = "cmd:" + k + ":" + i + ":" + j
-      )
+  private predicate cmdSubstitutionReplacement(string command, string id, int lineIndex) {
+    this.commandInSubstitution(lineIndex, command, id)
+    or
+    this.commandInBackticks(lineIndex, command, id)
+  }
+
+  /**
+   * Holds if there is a command substitution `$(command)` in
+   * the line at `lineIndex` in the shell script,
+   * and `id` is a unique identifier for this command.
+   */
+  private predicate commandInSubstitution(int lineIndex, string command, string id) {
+    exists(int occurrenceIndex, int occurrenceOffset |
+      command =
+        // Look for the command inside a $(...) command substitution
+        this.lineProducer(lineIndex)
+            .regexpFind("\\$\\((?:[^()]+|\\((?:[^()]+|\\([^()]*\\))*\\))*\\)", occurrenceIndex,
+              occurrenceOffset)
+            // trim starting $( - TODO do this in first regex
+            .regexpReplaceAll("^\\$\\(", "")
+            // trim ending ) - TODO do this in first regex
+            .regexpReplaceAll("\\)$", "") and
+      id = "cmdsubs:" + lineIndex + ":" + occurrenceIndex + ":" + occurrenceOffset
     )
   }
 
-  private predicate rankedCmdSubstitutionReplacements(int i, string old, string new) {
-    old = rank[i](string old2 | this.cmdSubstitutionReplacement(old2, _, _) | old2) and
-    this.cmdSubstitutionReplacement(old, new, _)
+  /**
+   * Holds if `command` is a command in backticks `` `...` `` in
+   * the line at `lineIndex` in the shell script,
+   * and `id` is a unique identifier for this command.
+   */
+  private predicate commandInBackticks(int lineIndex, string command, string id) {
+    exists(int occurrenceIndex, int occurrenceOffset |
+      command =
+        this.lineProducer(lineIndex)
+            .regexpFind("\\`[^\\`]+\\`", occurrenceIndex, occurrenceOffset)
+            // trim leading backtick - TODO do this in first regex
+            .regexpReplaceAll("^\\`", "")
+            // trim trailing backtick - TODO do this in first regex
+            .regexpReplaceAll("\\`$", "") and
+      id = "cmd:" + lineIndex + ":" + occurrenceIndex + ":" + occurrenceOffset
+    )
+  }
+
+  private predicate rankedCmdSubstitutionReplacements(int i, string command, string commandId) {
+    // rank commands by their unique IDs
+    commandId = rank[i](string c, string id | this.cmdSubstitutionReplacement(c, id, _) | id) and
+    // since we cannot output (command, ID) tuples from the rank operation,
+    // we need to work out the specific command associated with the resulting ID
+    this.cmdSubstitutionReplacement(command, commandId, _)
   }
 
   private predicate doReplaceCmdSubstitutions(int line, int round, string old, string new) {
@@ -64,31 +93,56 @@ class BashShellScript extends ShellScript {
     this.cmdSubstitutionReplacement(result, _, i)
   }
 
+  /**
+   * Holds if `quotedStr` is a string in double quotes in
+   * the line at `lineIndex` in the shell script,
+   * and `id` is a unique identifier for this quoted string.
+   */
+  private predicate doubleQuotedString(int lineIndex, string quotedStr, string id) {
+    exists(int occurrenceIndex, int occurrenceOffset |
+      // double quoted string
+      quotedStr =
+        this.cmdSubstitutedLineProducer(lineIndex)
+            .regexpFind("\"((?:[^\"\\\\]|\\\\.)*)\"", occurrenceIndex, occurrenceOffset) and
+      id =
+        "qstr:" + lineIndex + ":" + occurrenceIndex + ":" + occurrenceOffset + ":" +
+          quotedStr.length() + ":" + quotedStr.regexpReplaceAll("[^a-zA-Z0-9]", "")
+    )
+  }
+
+  /**
+   * Holds if `quotedStr` is a string in single quotes in
+   * the line at `lineIndex` in the shell script,
+   * and `id` is a unique identifier for this quoted string.
+   */
+  private predicate singleQuotedString(int lineIndex, string quotedStr, string id) {
+    exists(int occurrenceIndex, int occurrenceOffset |
+      // single quoted string
+      quotedStr =
+        this.cmdSubstitutedLineProducer(lineIndex)
+            .regexpFind("'((?:\\\\.|[^'\\\\])*)'", occurrenceIndex, occurrenceOffset) and
+      id =
+        "qstr:" + lineIndex + ":" + occurrenceIndex + ":" + occurrenceOffset + ":" +
+          quotedStr.length() + ":" + quotedStr.regexpReplaceAll("[^a-zA-Z0-9]", "")
+    )
+  }
+
   private predicate quotedStringReplacement(string quotedStr, string id) {
-    exists(string line, int k | line = this.cmdSubstitutedLineProducer(k) |
-      exists(int i, int j |
-        // double quoted string
-        quotedStr = line.regexpFind("\"((?:[^\"\\\\]|\\\\.)*)\"", i, j) and
-        id =
-          "qstr:" + k + ":" + i + ":" + j + ":" + quotedStr.length() + ":" +
-            quotedStr.regexpReplaceAll("[^a-zA-Z0-9]", "")
-      )
+    exists(int lineIndex |
+      this.doubleQuotedString(lineIndex, quotedStr, id)
       or
-      exists(int i, int j |
-        // single quoted string
-        quotedStr = line.regexpFind("'((?:\\\\.|[^'\\\\])*)'", i, j) and
-        id =
-          "qstr:" + k + ":" + i + ":" + j + ":" + quotedStr.length() + ":" +
-            quotedStr.regexpReplaceAll("[^a-zA-Z0-9]", "")
-      )
+      this.singleQuotedString(lineIndex, quotedStr, id)
     ) and
     // Only do this for strings that might otherwise disrupt subsequent parsing
     quotedStr.regexpMatch("[\"'].*[$\n\r'\"" + Bash::separator() + "].*[\"']")
   }
 
-  private predicate rankedQuotedStringReplacements(int i, string old, string new) {
-    old = rank[i](string old2 | this.quotedStringReplacement(old2, _) | old2) and
-    this.quotedStringReplacement(old, new)
+  private predicate rankedQuotedStringReplacements(int i, string quotedString, string quotedStringId) {
+    // rank quoted strings by their nearly-unique IDs
+    quotedStringId = rank[i](string s, string id | this.quotedStringReplacement(s, id) | id) and
+    // since we cannot output (string, ID) tuples from the rank operation,
+    // we need to work out the specific string associated with the resulting ID
+    this.quotedStringReplacement(quotedString, quotedStringId)
   }
 
   private predicate doReplaceQuotedStrings(int line, int round, string old, string new) {

actions/ql/lib/codeql/actions/security/OutputClobberingQuery.qll

@@ -214,6 +214,10 @@ private module OutputClobberingConfig implements DataFlow::ConfigSig {
       )
     )
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
+
+  Location getASelectedSourceLocation(DataFlow::Node sink) { none() }
 }
 
 /** Tracks flow of unsafe user input that is used to construct and evaluate an environment variable. */

actions/ql/lib/codeql/actions/security/RequestForgeryQuery.qll

@@ -16,6 +16,10 @@ private module RequestForgeryConfig implements DataFlow::ConfigSig {
   predicate isSource(DataFlow::Node source) { source instanceof RemoteFlowSource }
 
   predicate isSink(DataFlow::Node sink) { sink instanceof RequestForgerySink }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
+
+  Location getASelectedSourceLocation(DataFlow::Node sink) { none() }
 }
 
 /** Tracks flow of unsafe user input that is used to construct and evaluate a system command. */

actions/ql/lib/codeql/actions/security/SecretExfiltrationQuery.qll

@@ -15,6 +15,10 @@ private module SecretExfiltrationConfig implements DataFlow::ConfigSig {
   predicate isSource(DataFlow::Node source) { source instanceof RemoteFlowSource }
 
   predicate isSink(DataFlow::Node sink) { sink instanceof SecretExfiltrationSink }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
+
+  Location getASelectedSourceLocation(DataFlow::Node sink) { none() }
 }
 
 /** Tracks flow of unsafe user input that is used in a context where it may lead to a secret exfiltration. */

actions/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/actions-all
-version: 0.4.10
+version: 0.4.13-dev
 library: true
 warnOnImplicitThis: true
 dependencies:

actions/ql/src/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.6.4
+
+No user-facing changes.
+
+## 0.6.3
+
+No user-facing changes.
+
 ## 0.6.2
 
 ### Minor Analysis Improvements

actions/ql/src/Models/CompositeActionsSinks.ql

@@ -24,6 +24,10 @@ private module MyConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) {
     sink instanceof CodeInjectionSink and not madSink(sink, "code-injection")
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
+
+  Location getASelectedSourceLocation(DataFlow::Node sink) { none() }
 }
 
 module MyFlow = TaintTracking::Global<MyConfig>;

actions/ql/src/Models/CompositeActionsSources.ql

@@ -34,6 +34,10 @@ private module MyConfig implements DataFlow::ConfigSig {
     isSink(node) and
     set instanceof DataFlow::FieldContent
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
+
+  Location getASelectedSourceLocation(DataFlow::Node sink) { none() }
 }
 
 module MyFlow = TaintTracking::Global<MyConfig>;

actions/ql/src/Models/CompositeActionsSummaries.ql

@@ -25,6 +25,10 @@ private module MyConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) {
     exists(CompositeAction c | c.getAnOutputExpr() = sink.asExpr())
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
+
+  Location getASelectedSourceLocation(DataFlow::Node sink) { none() }
 }
 
 module MyFlow = TaintTracking::Global<MyConfig>;

actions/ql/src/Models/ReusableWorkflowsSinks.ql

@@ -24,6 +24,10 @@ private module MyConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) {
     sink instanceof CodeInjectionSink and not madSink(sink, "code-injection")
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
+
+  Location getASelectedSourceLocation(DataFlow::Node sink) { none() }
 }
 
 module MyFlow = TaintTracking::Global<MyConfig>;

actions/ql/src/Models/ReusableWorkflowsSources.ql

@@ -34,6 +34,10 @@ private module MyConfig implements DataFlow::ConfigSig {
     isSink(node) and
     set instanceof DataFlow::FieldContent
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
+
+  Location getASelectedSourceLocation(DataFlow::Node sink) { none() }
 }
 
 module MyFlow = TaintTracking::Global<MyConfig>;

actions/ql/src/Models/ReusableWorkflowsSummaries.ql

@@ -25,6 +25,10 @@ private module MyConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) {
     exists(ReusableWorkflow w | w.getAnOutputExpr() = sink.asExpr())
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
+
+  Location getASelectedSourceLocation(DataFlow::Node sink) { none() }
 }
 
 module MyFlow = TaintTracking::Global<MyConfig>;

actions/ql/src/Security/CWE-077/EnvPathInjectionCritical.md

@@ -1,6 +1,4 @@
-# Environment Path Injection
-
-## Description
+## Overview
 
 GitHub Actions allow to define the system PATH variable by writing to a file pointed to by the `GITHUB_PATH` environment variable. Writing to this file appends a directory to the system PATH variable and automatically makes it available to all subsequent actions in the current job.
 
@@ -12,11 +10,11 @@ echo "$HOME/.local/bin" >> $GITHUB_PATH
 
 If an attacker can control the contents of the system PATH, they are able to influence what commands are run in subsequent steps of the same job.
 
-## Recommendations
+## Recommendation
 
 Do not allow untrusted data to influence the system PATH: Avoid using untrusted data sources (e.g., artifact content) to define the system PATH.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -36,4 +34,4 @@ If an attacker can manipulate the value being set, such as through artifact down
 
 ## References
 
-- [Workflow commands for GitHub Actions](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions)
+- GitHub Docs: [Workflow commands for GitHub Actions](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions).

actions/ql/src/Security/CWE-077/EnvPathInjectionMedium.md

@@ -1,6 +1,4 @@
-# Environment Path Injection
-
-## Description
+## Overview
 
 GitHub Actions allow to define the system PATH variable by writing to a file pointed to by the `GITHUB_PATH` environment variable. Writing to this file appends a directory to the system PATH variable and automatically makes it available to all subsequent actions in the current job.
 
@@ -12,11 +10,11 @@ echo "$HOME/.local/bin" >> $GITHUB_PATH
 
 If an attacker can control the contents of the system PATH, they are able to influence what commands are run in subsequent steps of the same job.
 
-## Recommendations
+## Recommendation
 
 Do not allow untrusted data to influence the system PATH: Avoid using untrusted data sources (e.g., artifact content) to define the system PATH.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -36,4 +34,4 @@ If an attacker can manipulate the value being set, such as through artifact down
 
 ## References
 
-- [Workflow commands for GitHub Actions](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions)
+- GitHub Docs: [Workflow commands for GitHub Actions](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions).

actions/ql/src/Security/CWE-077/EnvVarInjectionCritical.md

@@ -1,6 +1,4 @@
-# Environment Variable Injection
-
-## Description
+## Overview
 
 GitHub Actions allow to define environment variables by writing to a file pointed to by the `GITHUB_ENV` environment variable:
 
@@ -37,7 +35,7 @@ steps:
 
 If an attacker can control the values assigned to environment variables and there is no sanitization in place, the attacker will be able to inject additional variables by injecting new lines or `{delimiters}`.
 
-## Recommendations
+## Recommendation
 
 1. **Do not allow untrusted data to influence environment variables**:
 
@@ -64,7 +62,7 @@ If an attacker can control the values assigned to environment variables and ther
           } >> "$GITHUB_ENV"
     ```
 
-## Examples
+## Example
 
 ### Example of Vulnerability
 
@@ -113,5 +111,5 @@ An attacker is be able to run arbitrary code by injecting environment variables
 
 ## References
 
-- [Workflow commands for GitHub Actions](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions)
-- [GitHub Actions Exploitation: Repo Jacking and Environment Manipulation](https://www.synacktiv.com/publications/github-actions-exploitation-repo-jacking-and-environment-manipulation)
+- GitHub Docs: [Workflow commands for GitHub Actions](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions).
+- Synacktiv: [GitHub Actions Exploitation: Repo Jacking and Environment Manipulation](https://www.synacktiv.com/publications/github-actions-exploitation-repo-jacking-and-environment-manipulation).

actions/ql/src/Security/CWE-077/EnvVarInjectionMedium.md

@@ -1,6 +1,4 @@
-# Environment Variable Injection
-
-## Description
+## Overview
 
 GitHub Actions allow to define environment variables by writing to a file pointed to by the `GITHUB_ENV` environment variable:
 
@@ -37,7 +35,7 @@ steps:
 
 If an attacker can control the values assigned to environment variables and there is no sanitization in place, the attacker will be able to inject additional variables by injecting new lines or `{delimiters}`.
 
-## Recommendations
+## Recommendation
 
 1. **Do not allow untrusted data to influence environment variables**:
 
@@ -64,7 +62,7 @@ If an attacker can control the values assigned to environment variables and ther
           } >> "$GITHUB_ENV"
     ```
 
-## Examples
+## Example
 
 ### Example of Vulnerability
 
@@ -113,5 +111,5 @@ An attacker would be able to run arbitrary code by injecting environment variabl
 
 ## References
 
-- [Workflow commands for GitHub Actions](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions)
-- [GitHub Actions Exploitation: Repo Jacking and Environment Manipulation](https://www.synacktiv.com/publications/github-actions-exploitation-repo-jacking-and-environment-manipulation)
+- GitHub Docs: [Workflow commands for GitHub Actions](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions).
+- Synacktiv: [GitHub Actions Exploitation: Repo Jacking and Environment Manipulation](https://www.synacktiv.com/publications/github-actions-exploitation-repo-jacking-and-environment-manipulation).

actions/ql/src/Security/CWE-094/CodeInjectionCritical.md

@@ -1,18 +1,16 @@
-# Code Injection in GitHub Actions
-
-## Description
+## Overview
 
 Using user-controlled input in GitHub Actions may lead to code injection in contexts like _run:_ or _script:_.
 
 Code injection in GitHub Actions may allow an attacker to exfiltrate any secrets used in the workflow and the temporary GitHub repository authorization token. The token may have write access to the repository, allowing an attacker to make changes to the repository.
 
-## Recommendations
+## Recommendation
 
 The best practice to avoid code injection vulnerabilities in GitHub workflows is to set the untrusted input value of the expression to an intermediate environment variable and then use the environment variable using the native syntax of the shell/script interpreter (that is, not _${{ env.VAR }}_).
 
 It is also recommended to limit the permissions of any tokens used by a workflow such as the GITHUB_TOKEN.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 

actions/ql/src/Security/CWE-094/CodeInjectionMedium.md

@@ -1,18 +1,16 @@
-# Code Injection in GitHub Actions
-
-## Description
+## Overview
 
 Using user-controlled input in GitHub Actions may lead to code injection in contexts like _run:_ or _script:_.
 
 Code injection in GitHub Actions may allow an attacker to exfiltrate any secrets used in the workflow and the temporary GitHub repository authorization token. The token may have write access to the repository, allowing an attacker to make changes to the repository.
 
-## Recommendations
+## Recommendation
 
 The best practice to avoid code injection vulnerabilities in GitHub workflows is to set the untrusted input value of the expression to an intermediate environment variable and then use the environment variable using the native syntax of the shell/script interpreter (that is, not _${{ env.VAR }}_).
 
 It is also recommended to limit the permissions of any tokens used by a workflow such as the GITHUB_TOKEN.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 

actions/ql/src/Security/CWE-1395/UseOfKnownVulnerableAction.md

@@ -1,13 +1,11 @@
-# Use of Actions with known vulnerabilities
-
-## Description
+## Overview
 
 The security of the workflow and the repository could be compromised by GitHub Actions workflows that utilize GitHub Actions with known vulnerabilities.
 
-## Recommendations
+## Recommendation
 
 Either remove the component from the workflow or upgrade it to a version that is not vulnerable.
 
 ## References
 
-- [GitHub Docs: Keeping your actions up to date with Dependabot](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot)
+- GitHub Docs: [Keeping your actions up to date with Dependabot](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot).

actions/ql/src/Security/CWE-275/MissingActionsPermissions.md

@@ -1,12 +1,21 @@
-# Actions Job and Workflow Permissions are not set
-
-## Description
+## Overview
 
 If a GitHub Actions job or workflow has no explicit permissions set, then the repository permissions are used. Repositories created under organizations inherit the organization permissions. The organizations or repositories created before February 2023 have the default permissions set to read-write. Often these permissions do not adhere to the principle of least privilege and can be reduced to read-only, leaving the `write` permission only to a specific types as `issues: write` or `pull-requests: write`.
 
-## Recommendations
+## Recommendation
 
-Add the `permissions` key to the job or the root of workflow (in this case it is applied to all jobs in the workflow that do not have their own `permissions` key) and assign the least privileges required to complete the task:
+Add the `permissions` key to the job or the root of workflow (in this case it is applied to all jobs in the workflow that do not have their own `permissions` key) and assign the least privileges required to complete the task.
+
+## Example
+
+### Incorrect Usage
+
+```yaml
+name: "My workflow"
+# No permissions block
+```
+
+### Correct Usage
 
 ```yaml
 name: "My workflow"
@@ -27,4 +36,4 @@ jobs:
 
 ## References
 
-- [Assigning permissions to jobs](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/assigning-permissions-to-jobs)
+- GitHub Docs: [Assigning permissions to jobs](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/assigning-permissions-to-jobs).

actions/ql/src/Security/CWE-285/ImproperAccessControl.md

@@ -1,14 +1,12 @@
-# Improper Access Control
-
-## Description
+## Overview
 
 Sometimes labels are used to approve GitHub Actions. An authorization check may not be properly implemented, allowing an attacker to mutate the code after it has been reviewed and approved by label.
 
-## Recommendations
+## Recommendation
 
 When using labels, make sure that the code cannot be modified after it has been reviewed and the label has been set.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -57,4 +55,4 @@ jobs:
 
 ## References
 
-- [Events that trigger workflows](https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#pull_request_target)
+- GitHub Docs: [Events that trigger workflows](https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#pull_request_target).

actions/ql/src/Security/CWE-312/ExcessiveSecretsExposure.md

@@ -1,14 +1,12 @@
-# Excessive Secrets Exposure
-
-## Description
+## Overview
 
 When the workflow runner cannot determine what secrets are needed to run the workflow, it will pass all the available secrets to the runner including organization and repository secrets. This violates the least privileged principle and increases the impact of a potential vulnerability affecting the workflow.
 
-## Recommendations
+## Recommendation
 
 Only pass those secrets that are needed by the workflow. Avoid using expressions such as `toJSON(secrets)` or dynamically accessed secrets such as `secrets[format('GH_PAT_%s', matrix.env)]` since the workflow will need to receive all secrets to decide at runtime which one needs to be used.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -48,5 +46,5 @@ env:
 
 ## References
 
-- [Using secrets in GitHub Actions](https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions#using-encrypted-secrets-in-a-workflow)
-- [Job uses all secrets](https://github.com/boostsecurityio/poutine/blob/main/docs/content/en/rules/job_all_secrets.md)
+- GitHub Docs: [Using secrets in GitHub Actions](https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions#using-encrypted-secrets-in-a-workflow).
+- poutine: [Job uses all secrets](https://github.com/boostsecurityio/poutine/blob/main/docs/content/en/rules/job_all_secrets.md).

actions/ql/src/Security/CWE-312/SecretsInArtifacts.md

@@ -1,6 +1,4 @@
-# Storage of sensitive information in GitHub Actions artifact
-
-## Description
+## Overview
 
 Sensitive information included in a GitHub Actions artifact can allow an attacker to access the sensitive information if the artifact is published.
 
@@ -10,6 +8,8 @@ Only store information that is meant to be publicly available in a GitHub Action
 
 ## Example
 
+### Incorrect Usage
+
 The following example uses `actions/checkout` to checkout code which stores the GITHUB_TOKEN in the \`.git/config\` file and then stores the contents of the \`.git\` repository into the artifact:
 
 ```yaml
@@ -28,6 +28,8 @@ jobs:
           path: .
 ```
 
+### Correct Usage
+
 The issue has been fixed below, where the `actions/upload-artifact` uses a version (v4+) which does not include hidden files or directories into the artifact.
 
 ```yaml

actions/ql/src/Security/CWE-312/UnmaskedSecretExposure.md

@@ -1,14 +1,12 @@
-# Unmasked Secret Exposure
-
-## Description
+## Overview
 
 Secrets derived from other secrets are not known to the workflow runner, and therefore are not masked unless explicitly registered.
 
-## Recommendations
+## Recommendation
 
 Avoid defining non-plain secrets. For example, do not define a new secret containing a JSON object and then read properties out of it from the workflow, since these read values will not be masked by the workflow runner.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -34,4 +32,4 @@ Avoid defining non-plain secrets. For example, do not define a new secret contai
 
 ## References
 
-- [Using secrets in GitHub Actions](https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions#using-encrypted-secrets-in-a-workflow)
+- GitHub Docs: [Using secrets in GitHub Actions](https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions#using-encrypted-secrets-in-a-workflow).

actions/ql/src/Security/CWE-349/CachePoisoningViaCodeInjection.md

@@ -1,6 +1,4 @@
-# Cache Poisoning in GitHub Actions
-
-## Description
+## Overview
 
 GitHub Actions cache poisoning is a technique that allows an attacker to inject malicious content into the Action's cache from unprivileged workflow, potentially leading to code execution in privileged workflows.
 
@@ -23,7 +21,7 @@ In GitHub Actions, cache scopes are primarily determined by the branch structure
 
 Due to the above design, if something is cached in the context of the default branch (e.g., `main`), it becomes accessible to any feature branch derived from `main`.
 
-## Recommendations
+## Recommendation
 
 1. Avoid using caching in workflows that handle sensitive operations like releases.
 2. If caching must be used:
@@ -34,7 +32,7 @@ Due to the above design, if something is cached in the context of the default br
 4. Never run untrusted code in the context of the default branch.
 5. Sign the cache value cryptographically and verify the signature before usage.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -78,6 +76,6 @@ jobs:
 
 ## References
 
-- [The Monsters in Your Build Cache – GitHub Actions Cache Poisoning](https://adnanthekhan.com/2024/05/06/the-monsters-in-your-build-cache-github-actions-cache-poisoning/)
-- [GitHub Actions Caching Documentation](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows)
-- [Cache Poisoning in GitHub Actions](https://scribesecurity.com/blog/github-cache-poisoning/)
+- Adnan Khan's Blog: [The Monsters in Your Build Cache – GitHub Actions Cache Poisoning](https://adnanthekhan.com/2024/05/06/the-monsters-in-your-build-cache-github-actions-cache-poisoning/).
+- GitHub Docs: [GitHub Actions Caching Documentation](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows).
+- Scribe Security Blog: [Cache Poisoning in GitHub Actions](https://scribesecurity.com/blog/github-cache-poisoning/).

actions/ql/src/Security/CWE-349/CachePoisoningViaDirectCache.md

@@ -1,6 +1,4 @@
-# Cache Poisoning in GitHub Actions
-
-## Description
+## Overview
 
 GitHub Actions cache poisoning is a technique that allows an attacker to inject malicious content into the Action's cache from unprivileged workflow, potentially leading to code execution in privileged workflows.
 
@@ -23,7 +21,7 @@ In GitHub Actions, cache scopes are primarily determined by the branch structure
 
 Due to the above design, if something is cached in the context of the default branch (e.g., `main`), it becomes accessible to any feature branch derived from `main`.
 
-## Recommendations
+## Recommendation
 
 1. Avoid using caching in workflows that handle sensitive operations like releases.
 2. If caching must be used:
@@ -34,7 +32,7 @@ Due to the above design, if something is cached in the context of the default br
 4. Never run untrusted code in the context of the default branch.
 5. Sign the cache value cryptographically and verify the signature before usage.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -123,6 +121,6 @@ jobs:
 
 ## References
 
-- [The Monsters in Your Build Cache – GitHub Actions Cache Poisoning](https://adnanthekhan.com/2024/05/06/the-monsters-in-your-build-cache-github-actions-cache-poisoning/)
-- [GitHub Actions Caching Documentation](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows)
-- [Cache Poisoning in GitHub Actions](https://scribesecurity.com/blog/github-cache-poisoning/)
+- Adnan Khan's Blog: [The Monsters in Your Build Cache – GitHub Actions Cache Poisoning](https://adnanthekhan.com/2024/05/06/the-monsters-in-your-build-cache-github-actions-cache-poisoning/).
+- GitHub Docs: [GitHub Actions Caching Documentation](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows).
+- Scribe Security Blog: [Cache Poisoning in GitHub Actions](https://scribesecurity.com/blog/github-cache-poisoning/).

actions/ql/src/Security/CWE-349/CachePoisoningViaPoisonableStep.md

@@ -1,6 +1,4 @@
-# Cache Poisoning in GitHub Actions
-
-## Description
+## Overview
 
 GitHub Actions cache poisoning is a technique that allows an attacker to inject malicious content into the Action's cache from unprivileged workflow, potentially leading to code execution in privileged workflows.
 
@@ -23,7 +21,7 @@ In GitHub Actions, cache scopes are primarily determined by the branch structure
 
 Due to the above design, if something is cached in the context of the default branch (e.g., `main`), it becomes accessible to any feature branch derived from `main`.
 
-## Recommendations
+## Recommendation
 
 1. Avoid using caching in workflows that handle sensitive operations like releases.
 2. If caching must be used:
@@ -34,7 +32,7 @@ Due to the above design, if something is cached in the context of the default br
 4. Never run untrusted code in the context of the default branch.
 5. Sign the cache value cryptographically and verify the signature before usage.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -80,6 +78,6 @@ jobs:
 
 ## References
 
-- [The Monsters in Your Build Cache – GitHub Actions Cache Poisoning](https://adnanthekhan.com/2024/05/06/the-monsters-in-your-build-cache-github-actions-cache-poisoning/)
-- [GitHub Actions Caching Documentation](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows)
-- [Cache Poisoning in GitHub Actions](https://scribesecurity.com/blog/github-cache-poisoning/)
+- Adnan Khan's Blog: [The Monsters in Your Build Cache – GitHub Actions Cache Poisoning](https://adnanthekhan.com/2024/05/06/the-monsters-in-your-build-cache-github-actions-cache-poisoning/).
+- GitHub Docs: [GitHub Actions Caching Documentation](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows).
+- Scribe Security Blog: [Cache Poisoning in GitHub Actions](https://scribesecurity.com/blog/github-cache-poisoning/).

actions/ql/src/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.md

@@ -1,17 +1,15 @@
-# Untrusted Checkout TOCTOU (Time-of-check to time-of-use)
-
-## Description
+## Overview
 
 Untrusted Checkout is protected by a security check but the checked-out branch can be changed after the check.
 
-## Recommendations
+## Recommendation
 
 Verify that the code has not been modified after the security check. This may be achieved differently depending on the type of check:
 
 - Deployment Environment Approval: Make sure to use a non-mutable reference to the code to be executed. For example use a `sha` instead of a `ref`.
 - Label Gates: Make sure to use a non-mutable reference to the code to be executed. For example use a `sha` instead of a `ref`.
 
-## Examples
+## Example
 
 ### Incorrect Usage (Deployment Environment Approval)
 
@@ -99,4 +97,4 @@ jobs:
 
 ## References
 
-- [ActionsTOCTOU](https://github.com/AdnaneKhan/ActionsTOCTOU)
+- [ActionsTOCTOU](https://github.com/AdnaneKhan/ActionsTOCTOU).

actions/ql/src/Security/CWE-367/UntrustedCheckoutTOCTOUHigh.md

@@ -1,17 +1,15 @@
-# Untrusted Checkout TOCTOU (Time-of-check to time-of-use)
-
-## Description
+## Overview
 
 Untrusted Checkout is protected by a security check but the checked-out branch can be changed after the check.
 
-## Recommendations
+## Recommendation
 
 Verify that the code has not been modified after the security check. This may be achieved differently depending on the type of check:
 
 - Deployment Environment Approval: Make sure to use a non-mutable reference to the code to be executed. For example use a `sha` instead of a `ref`.
 - Label Gates: Make sure to use a non-mutable reference to the code to be executed. For example use a `sha` instead of a `ref`.
 
-## Examples
+## Example
 
 ### Incorrect Usage (Deployment Environment Approval)
 
@@ -99,4 +97,4 @@ jobs:
 
 ## References
 
-- [ActionsTOCTOU](https://github.com/AdnaneKhan/ActionsTOCTOU)
+- [ActionsTOCTOU](https://github.com/AdnaneKhan/ActionsTOCTOU).

actions/ql/src/Security/CWE-571/ExpressionIsAlwaysTrueCritical.md

@@ -1,6 +1,4 @@
-# If Condition Always Evaluates to True
-
-## Description
+## Overview
 
 GitHub Workflow Expressions (`${{ ... }}`) used in the `if` condition of jobs or steps must not contain extra characters or spaces. Otherwise, the condition is invariably evaluated to `true`.
 
@@ -14,7 +12,7 @@ To avoid the vulnerability where an `if` condition always evaluates to `true`, i
 2. Avoid multiline or spaced-out conditional expressions that might inadvertently introduce unwanted characters or formatting.
 3. Test the workflow to ensure the `if` conditions behave as expected under different scenarios.
 
-## Examples
+## Example
 
 ### Correct Usage
 
@@ -60,4 +58,4 @@ To avoid the vulnerability where an `if` condition always evaluates to `true`, i
 
 ## References
 
-- [Expression Always True Github Issue](https://github.com/actions/runner/issues/1173)
+- GitHub actions/runner Issues: [Expression Always True](https://github.com/actions/runner/issues/1173).

actions/ql/src/Security/CWE-571/ExpressionIsAlwaysTrueHigh.md

@@ -1,6 +1,4 @@
-# If Condition Always Evaluates to True
-
-## Description
+## Overview
 
 GitHub Workflow Expressions (`${{ ... }}`) used in the `if` condition of jobs or steps must not contain extra characters or spaces. Otherwise, the condition is invariably evaluated to `true`.
 
@@ -14,7 +12,7 @@ To avoid the vulnerability where an `if` condition always evaluates to `true`, i
 2. Avoid multiline or spaced-out conditional expressions that might inadvertently introduce unwanted characters or formatting.
 3. Test the workflow to ensure the `if` conditions behave as expected under different scenarios.
 
-## Examples
+## Example
 
 ### Correct Usage
 
@@ -60,4 +58,4 @@ To avoid the vulnerability where an `if` condition always evaluates to `true`, i
 
 ## References
 
-- [Expression Always True Github Issue](https://github.com/actions/runner/issues/1173)
+- GitHub actions/runner Issues: [Expression Always True](https://github.com/actions/runner/issues/1173).

actions/ql/src/Security/CWE-829/ArtifactPoisoningCritical.md

@@ -1,16 +1,14 @@
-# Artifact poisoning
-
-## Description
+## Overview
 
 The workflow downloads artifacts that may be poisoned by an attacker in previously triggered workflows. If the contents of these artifacts are not correctly extracted, stored and verified, they may lead to repository compromise if untrusted code gets executed in a privileged job.
 
-## Recommendations
+## Recommendation
 
 - Always consider artifacts content as untrusted.
 - Extract the contents of artifacts to a temporary folder so they cannot override existing files.
 - Verify the contents of the artifacts downloaded. If an artifact is expected to contain a numeric value, verify it before using it.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -69,4 +67,4 @@ jobs:
 
 ## References
 
-- [Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/)
+- GitHub Security Lab Research: [Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/).

actions/ql/src/Security/CWE-829/ArtifactPoisoningMedium.md

@@ -1,16 +1,14 @@
-# Artifact poisoning
-
-## Description
+## Overview
 
 The workflow downloads artifacts that may be poisoned by an attacker in previously triggered workflows. If the contents of these artifacts are not correctly extracted, stored and verified, they may lead to repository compromise if untrusted code gets executed in a privileged job.
 
-## Recommendations
+## Recommendation
 
 - Always consider artifacts content as untrusted.
 - Extract the contents of artifacts to a temporary folder so they cannot override existing files.
 - Verify the contents of the artifacts downloaded. If an artifact is expected to contain a numeric value, verify it before using it.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -69,4 +67,4 @@ jobs:
 
 ## References
 
-- [Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/)
+- GitHub Security Lab Research: [Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/).

actions/ql/src/Security/CWE-829/UnpinnedActionsTag.md

@@ -1,14 +1,12 @@
-# Unpinned tag for 3rd party Action in workflow
-
-## Description
+## Overview
 
 Using a tag for a 3rd party Action that is not pinned to a commit can lead to executing an untrusted Action through a supply chain attack.
 
-## Recommendations
+## Recommendation
 
 Pinning an action to a full length commit SHA is currently the only way to use a non-immutable action as an immutable release. Pinning to a particular SHA helps mitigate the risk of a bad actor adding a backdoor to the action's repository, as they would need to generate a SHA-1 collision for a valid Git object payload. When selecting a SHA, you should verify it is from the action's repository and not a repository fork.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -24,4 +22,4 @@ Pinning an action to a full length commit SHA is currently the only way to use a
 
 ## References
 
-- [Using third-party actions](https://docs.github.com/en/actions/security-for-github-actions/security-guides/security-hardening-for-github-actions#using-third-party-actions)
+- GitHub Docs: [Using third-party actions](https://docs.github.com/en/actions/security-for-github-actions/security-guides/security-hardening-for-github-actions#using-third-party-actions).

actions/ql/src/Security/CWE-829/UntrustedCheckoutCritical.md

@@ -1,10 +1,8 @@
-# Execution of Untrusted Checked-out Code
-
-## Description
+## Overview
 
 GitHub workflows can be triggered through various repository events, including incoming pull requests (PRs) or comments on Issues/PRs. A potentially dangerous misuse of the triggers such as `pull_request_target` or `issue_comment` followed by an explicit checkout of untrusted code (Pull Request HEAD) may lead to repository compromise if untrusted code gets executed in a privileged job.
 
-## Recommendations
+## Recommendation
 
 - Avoid using `pull_request_target` unless necessary.
 - Employ unprivileged `pull_request` workflows followed by `workflow_run` for privileged operations.
@@ -14,7 +12,7 @@ The best practice is to handle the potentially untrusted pull request via the **
 
 The artifacts downloaded from the first workflow should be considered untrusted and must be verified.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -134,4 +132,4 @@ jobs:
 
 ## References
 
-- [Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/)
+- GitHub Security Lab Research: [Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/).

actions/ql/src/Security/CWE-829/UntrustedCheckoutHigh.md

@@ -1,10 +1,8 @@
-# Execution of Untrusted Checked-out Code
-
-## Description
+## Overview
 
 GitHub workflows can be triggered through various repository events, including incoming pull requests (PRs) or comments on Issues/PRs. A potentially dangerous misuse of the triggers such as `pull_request_target` or `issue_comment` followed by an explicit checkout of untrusted code (Pull Request HEAD) may lead to repository compromise if untrusted code gets executed in a privileged job.
 
-## Recommendations
+## Recommendation
 
 - Avoid using `pull_request_target` unless necessary.
 - Employ unprivileged `pull_request` workflows followed by `workflow_run` for privileged operations.
@@ -14,7 +12,7 @@ The best practice is to handle the potentially untrusted pull request via the **
 
 The artifacts downloaded from the first workflow should be considered untrusted and must be verified.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -134,4 +132,4 @@ jobs:
 
 ## References
 
-- [Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/)
+- GitHub Security Lab Research: [Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/).

actions/ql/src/Security/CWE-829/UntrustedCheckoutMedium.md

@@ -1,10 +1,8 @@
-# Execution of Untrusted Checked-out Code
-
-## Description
+## Overview
 
 GitHub workflows can be triggered through various repository events, including incoming pull requests (PRs) or comments on Issues/PRs. A potentially dangerous misuse of the triggers such as `pull_request_target` or `issue_comment` followed by an explicit checkout of untrusted code (Pull Request HEAD) may lead to repository compromise if untrusted code gets executed in a privileged job.
 
-## Recommendations
+## Recommendation
 
 - Avoid using `pull_request_target` unless necessary.
 - Employ unprivileged `pull_request` workflows followed by `workflow_run` for privileged operations.
@@ -14,7 +12,7 @@ The best practice is to handle the potentially untrusted pull request via the **
 
 The artifacts downloaded from the first workflow should be considered untrusted and must be verified.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -134,4 +132,4 @@ jobs:
 
 ## References
 
-- [Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/)
+- GitHub Security Lab Research: [Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/).

actions/ql/src/Violations Of Best Practice/CodeQL/UnnecessaryUseOfAdvancedConfig.md

@@ -1,13 +1,11 @@
-# Unneccesary use of advanced configuration
-
-## Description
+## Overview
 
 The CodeQL workflow does not use any custom settings and could be simplified by switching to the CodeQL default setup.
 
-## Recommendations
+## Recommendation
 
 If there is no reason to have a custom configuration switch to the CodeQL default setup.
 
 ## References
 
-- [GitHub Docs: Configuring Default Setup for a repository](https://docs.github.com/en/code-security/code-scanning/enabling-code-scanning/configuring-default-setup-for-code-scanning#configuring-default-setup-for-a-repository)
+- GitHub Docs: [Configuring Default Setup for a repository](https://docs.github.com/en/code-security/code-scanning/enabling-code-scanning/configuring-default-setup-for-code-scanning#configuring-default-setup-for-a-repository).

actions/ql/src/change-notes/released/0.6.3.md

@@ -0,0 +1,3 @@
+## 0.6.3
+
+No user-facing changes.

actions/ql/src/change-notes/released/0.6.4.md

@@ -0,0 +1,3 @@
+## 0.6.4
+
+No user-facing changes.

actions/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.6.2
+lastReleaseVersion: 0.6.4

actions/ql/src/codeql-suites/actions-code-quality-extended.qls

@@ -0,0 +1,3 @@
+- queries: .
+- apply: code-quality-extended-selectors.yml
+  from: codeql/suite-helpers

actions/ql/src/experimental/Security/CWE-088/ArgumentInjectionCritical.md

@@ -1,18 +1,16 @@
-# Argument Injection in GitHub Actions
-
-## Description
+## Overview
 
 Passing user-controlled arguments to certain commands in the context of `Run` steps may lead to arbitrary code execution.
 
 Argument injection in GitHub Actions may allow an attacker to exfiltrate any secrets used in the workflow and the temporary GitHub repository authorization token. The token may have write access to the repository, allowing the attacker to make changes to the repository.
 
-## Recommendations
+## Recommendation
 
 When possible avoid passing user-controlled data to commands which may spawn new processes using some of their arguments.
 
 It is also recommended to limit the permissions of any tokens used by a workflow such as the GITHUB_TOKEN.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -35,7 +33,7 @@ An attacker may set the body of an Issue comment to `BAR/g;1e whoami;#` and the
 
 ## References
 
-- [Common Weakness Enumeration: CWE-88](https://cwe.mitre.org/data/definitions/88.html).
-- [Argument Injection Explained](https://sonarsource.github.io/argument-injection-vectors/explained/)
-- [Argument Injection Vectors](https://sonarsource.github.io/argument-injection-vectors/)
-- [GTFOBins](https://gtfobins.github.io/)
+- Common Weakness Enumeration: [CWE-88](https://cwe.mitre.org/data/definitions/88.html).
+- [Argument Injection Vectors](https://sonarsource.github.io/argument-injection-vectors/).
+- Argument Injection Vectors: [Argument Injection Explained](https://sonarsource.github.io/argument-injection-vectors/explained/).
+- [GTFOBins](https://gtfobins.github.io/).

actions/ql/src/experimental/Security/CWE-088/ArgumentInjectionMedium.md

@@ -1,18 +1,16 @@
-# Argument Injection in GitHub Actions
-
-## Description
+## Overview
 
 Passing user-controlled arguments to certain commands in the context of `Run` steps may lead to arbitrary code execution.
 
 Argument injection in GitHub Actions may allow an attacker to exfiltrate any secrets used in the workflow and the temporary GitHub repository authorization token. The token may have write access to the repository, allowing the attacker to make changes to the repository.
 
-## Recommendations
+## Recommendation
 
 When possible avoid passing user-controlled data to commands which may spawn new processes using some of their arguments.
 
 It is also recommended to limit the permissions of any tokens used by a workflow such as the GITHUB_TOKEN.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -35,7 +33,7 @@ An attacker may set the body of an Issue comment to `BAR|g;1e whoami;#` and the
 
 ## References
 
-- [Common Weakness Enumeration: CWE-88](https://cwe.mitre.org/data/definitions/88.html).
-- [Argument Injection Explained](https://sonarsource.github.io/argument-injection-vectors/explained/)
-- [Argument Injection Vectors](https://sonarsource.github.io/argument-injection-vectors/)
-- [GTFOBins](https://gtfobins.github.io/)
+- Common Weakness Enumeration: [CWE-88](https://cwe.mitre.org/data/definitions/88.html).
+- [Argument Injection Vectors](https://sonarsource.github.io/argument-injection-vectors/).
+- Argument Injection Vectors: [Argument Injection Explained](https://sonarsource.github.io/argument-injection-vectors/explained/).
+- [GTFOBins](https://gtfobins.github.io/).

actions/ql/src/experimental/Security/CWE-829/UnversionedImmutableAction.md

@@ -1,14 +1,12 @@
-# Unversioned Immutable Action
-
-## Description
+## Overview
 
 This action is eligible for Immutable Actions, a new GitHub feature that is currently only available for internal users. Immutable Actions are released as packages in the GitHub package registry instead of resolved from a pinned SHA at the repository. The Immutable Action provides the same immutability as pinning the version to a SHA but with improved readability and additional security guarantees.
 
-## Recommendations
+## Recommendation
 
 For internal users: when using [immutable actions](https://github.com/github/package-registry-team/blob/main/docs/immutable-actions/immutable-actions-howto.md) use the full semantic version of the action. This will ensure that the action is resolved to the exact version stored in the GitHub package registry.
 
-## Examples
+## Example
 
 ### Incorrect Usage
 
@@ -25,4 +23,4 @@ For internal users: when using [immutable actions](https://github.com/github/pac
 
 ## References
 
-- [Consuming immutable actions]()
+- [Consuming immutable actions]().

actions/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/actions-queries
-version: 0.6.2
+version: 0.6.5-dev
 library: false
 warnOnImplicitThis: true
 groups: [actions, queries]

actions/ql/test/query-tests/Security/CWE-094/.github/workflows/interpolation.yml

@@ -0,0 +1,81 @@
+name: Workflow with complex interpolation
+on: 
+  workflow_dispatch:
+    inputs:
+      choice-a:
+        required: true
+        type: choice
+        description: choice-a
+        default: a1
+        options: 
+        - a1
+        - a2
+        - a3
+      string-b:
+        required: false
+        type: string
+        description: string-b
+      string-c:
+        required: false
+        type: string
+        description: string-c
+      list-d:
+        required: true
+        type: string
+        default: d1 d2
+        description: list-d whitespace separated
+      list-e:
+        required: false
+        type: string
+        description: list-e whitespace separated
+      choice-f:
+        required: true
+        type: choice
+        description: choice-f
+        options: 
+        - false
+        - true
+          
+env:
+  DRY_TEST: false
+  B: ${{ github.event.inputs.string-b }}
+
+jobs:
+  job:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Produce values
+      id: produce-values
+      run: |
+        echo "region=region" >> $GITHUB_OUTPUT
+        echo "zone=zone" >> $GITHUB_OUTPUT
+
+    - name: Step with complex interpolation
+      id: complex
+      env:
+        CHOICE_A: ${{ github.event.inputs.choice-a }}
+        STRING_B: ${{ github.event.inputs.string-b }}
+        STRING_C: ${{ github.event.inputs.string-c }}
+        LIST_D: ${{ github.event.inputs.list-d }}
+        LIST_E: ${{ github.event.inputs.list-e }}
+        CHOICE_F: ${{ github.event.inputs.choice-f }}
+        REGION: ${{ steps.produce-values.outputs.region }}
+        ZONE: ${{ steps.produce-values.outputs.zone }}
+        DRY_TEST_JSON: ${{ fromJSON(env.DRY_TEST) }}
+        FUNCTION_NAME: my-function
+        USER_EMAIL: 'example@example.com'
+        TYPE: type
+        RANGE: '0-100'
+
+      run: |
+        comma_separated_list_d=$(echo "${LIST_D}" | sed "s/ /\",\"/g")
+        comma_separated_list_e=$(echo "${LIST_E}" | sed "s/ /\",\"/g")
+        c1=$(echo "${STRING_C}" | cut -d "-" -f 1)
+        c2=$(echo "${STRING_C}" | cut -d "-" -f 2)
+        # Similar commands that use JSON payloads with string interpolation.
+        response=$(aws lambda invoke --invocation-type RequestResponse --function-name "${FUNCTION_NAME}" --region "${REGION}" --cli-read-timeout 0 --cli-binary-format raw-in-base64-out --payload '{"appName":"my-app","chA":"'"${CHOICE_A}"'","c1":"'"${c1}"'","c2":"'"${c2}"'","a":"${CHOICE_A}","bValue":"${B}","zone":"${ZONE}","userEmail":"'"${USER_EMAIL}"'","region":"${REGION}","range":"${RANGE}","type":"${TYPE}","b":"${STRING_B}","listD":"","listE":"","dryTest":'"${DRY_TEST_JSON}"',"f":"${CHOICE_F}"}' ./config.json --log-type Tail)
+        response=$(aws lambda invoke --invocation-type RequestResponse --function-name "${FUNCTION_NAME}" --region "${REGION}" --cli-read-timeout 0 --cli-binary-format raw-in-base64-out --payload '{"appName":"my-app","chA":"'"${CHOICE_A}"'","c1":"'"${c1}"'","c2":"'"${c2}"'","a":"${CHOICE_A}","bValue":"${B}","zone":"${ZONE}","userEmail":"'"${USER_EMAIL}"'","region":"${REGION}","range":"${RANGE}","type":"${TYPE}","b":"${STRING_B}","listD":["'"${comma_separated_list_d}"'"],"listE":"","dryTest":'"${DRY_TEST_JSON}"',"f":"${CHOICE_F}"}' ./config.json --log-type Tail)
+        response=$(aws lambda invoke --invocation-type RequestResponse --function-name "${FUNCTION_NAME}" --region "${REGION}" --cli-read-timeout 0 --cli-binary-format raw-in-base64-out --payload '{"appName":"my-app","chA":"'"${CHOICE_A}"'","c1":"'"${c1}"'","c2":"'"${c2}"'","a":"${CHOICE_A}","bValue":"${B}","zone":"${ZONE}","userEmail":"'"${USER_EMAIL}"'","region":"${REGION}","range":"${RANGE}","type":"${TYPE}","b":"${STRING_B}","listD":["'"${comma_separated_list_d}"'"],"listE":"","dryTest":'"${DRY_TEST_JSON}"',"f":"${CHOICE_F}"}' ./config.json --log-type Tail)
+        response=$(aws lambda invoke --invocation-type RequestResponse --function-name "${FUNCTION_NAME}" --region "${REGION}" --cli-read-timeout 0 --cli-binary-format raw-in-base64-out --payload '{"appName":"my-app","chA":"'"${CHOICE_A}"'","c1":"'"${c1}"'","c2":"'"${c2}"'","a":"${CHOICE_A}","bValue":"${B}","zone":"${ZONE}","userEmail":"'"${USER_EMAIL}"'","region":"${REGION}","range":"${RANGE}","type":"${TYPE}","b":"${STRING_B}","listD":["'"${comma_separated_list_d}"'"],"listE":"","dryTest":'"${DRY_TEST_JSON}"',"f":"${CHOICE_F}"}' ./config.json --log-type Tail)
+        response=$(aws lambda invoke --invocation-type RequestResponse --function-name "${FUNCTION_NAME}" --region "${REGION}" --cli-read-timeout 0 --cli-binary-format raw-in-base64-out --payload '{"appName":"my-app","chA":"'"${CHOICE_A}"'","c1":"'"${c1}"'","c2":"'"${c2}"'","a":"${CHOICE_A}","bValue":"${B}","zone":"${ZONE}","userEmail":"'"${USER_EMAIL}"'","region":"${REGION}","range":"${RANGE}","type":"${TYPE}","b":"${STRING_B}","listD":"","listE":["'"${comma_separated_list_e}"'"],"dryTest":'"${DRY_TEST_JSON}"',"f":"${CHOICE_F}"}' ./config.json --log-type Tail)
+      shell: bash

config/add-overlay-annotations.py

@@ -0,0 +1,274 @@
+# This script is used to annotate .qll files without any existing overlay annotations
+# with overlay[local?] and overlay[caller?] annotations. Maintenance of overlay annotations
+# in annotated files will be handled by QL-for-QL queries.
+
+# It will walk the directory tree and annotate most .qll files, skipping only
+# some specific cases (e.g., empty files, files that configure dataflow for queries).
+
+# The script takes a list of languages and processes the corresponding directories.
+# If the optional --check argument is provided, the script checks for missing annotations,
+# but does not modify any files.
+
+# Usage: python3 add-overlay-annotations.py [--check] <language1> <language2> ...
+
+# The script will modify the files in place and print the changes made.
+# The script is designed to be run from the root of the repository.
+
+#!/usr/bin/python3
+import sys
+import os
+import re
+from difflib import context_diff
+
+OVERLAY_PATTERN = re.compile(r'overlay\[[a-zA-Z?_-]+\]')
+
+def has_overlay_annotations(lines):
+    '''
+    Check whether the given lines contain any overlay[...] annotations.
+    '''
+    return any(OVERLAY_PATTERN.search(line) for line in lines)
+
+
+def is_line_comment(line):
+    return line.startswith("//") or (line.startswith("/*") and line.endswith("*/"))
+
+
+def find_file_level_module_declaration(lines):
+    '''
+    Returns the index of the existing file-level module declaration if one
+    exists. Returns None otherwise.
+    '''
+    comment = False
+    for i, line in enumerate(lines):
+        trimmed = line.strip()
+
+        if is_line_comment(trimmed):
+            continue
+        elif trimmed.startswith("/*"):
+            comment = True
+        elif comment and trimmed.endswith("*/"):
+            comment = False
+        elif not comment and trimmed.endswith("module;"):
+            return i
+
+    return None
+
+
+def is_file_module_qldoc(i, lines):
+    '''
+    Assuming a qldoc ended on line i, determine if it belongs to the implicit
+    file-level module. If it is followed by another qldoc or imports, then it
+    does and if it is followed by any other non-empty, non-comment lines, then
+    we assume that is a declaration of some kind and the qldoc is attached to
+    that declaration.
+    '''
+    comment = False
+
+    for line in lines[i+1:]:
+        trimmed = line.strip()
+
+        if trimmed.startswith("import ") or trimmed.startswith("private import ") or trimmed.startswith("/**"):
+             return True
+        elif is_line_comment(trimmed) or not trimmed:
+             continue
+        elif trimmed.startswith("/*"):
+             comment = True
+        elif comment and trimmed.endswith("*/"):
+             comment = False
+        elif not comment and trimmed:
+             return False
+
+    return True
+
+
+def find_file_module_qldoc_declaration(lines):
+    '''
+    Returns the index of last line of the implicit file module qldoc if one
+    exists. Returns None otherwise.
+    '''
+
+    qldoc = False
+    comment = False
+    for i, line in enumerate(lines):
+        trimmed = line.strip()
+
+        if trimmed.startswith("//"):
+            continue
+        elif (qldoc or trimmed.startswith("/**")) and trimmed.endswith("*/"):
+            # a qldoc just ended; determine if it belongs to the implicit file module
+            if is_file_module_qldoc(i, lines):
+                return i
+            else:
+                return None
+        elif trimmed.startswith("/**"):
+            qldoc = True
+        elif trimmed.startswith("/*"):
+            comment = True
+        elif comment and trimmed.endswith("*/"):
+            comment = False
+        elif (not qldoc and not comment) and trimmed:
+            return None
+
+    return None
+
+
+def only_comments(lines):
+    '''
+    Returns true if the lines contain only comments and empty lines.
+    '''
+    comment = False
+
+    for line in lines:
+        trimmed = line.strip()
+
+        if not trimmed or is_line_comment(trimmed):
+            continue
+        elif trimmed.startswith("/*"):
+            comment = True
+        elif comment and trimmed.endswith("*/"):
+            comment = False
+        elif comment:
+            continue
+        elif trimmed:
+            return False
+
+    return True
+
+
+def insert_toplevel_maybe_local_annotation(filename, lines):
+    '''
+    Find a suitable place to insert an overlay[local?] annotation at the top of the file.
+    Returns a pair consisting of description and the modified lines or None if no overlay
+    annotation is necessary (e.g., for files that only contain comments).
+    '''
+    if only_comments(lines):
+        return None
+
+    i = find_file_level_module_declaration(lines)
+    if not i == None:
+        out_lines = lines[:i]
+        out_lines.append("overlay[local?]\n")
+        out_lines.extend(lines[i:])
+        return (f"Annotating \"{filename}\" via existing file-level module statement", out_lines)
+
+    i = find_file_module_qldoc_declaration(lines)
+    if not i == None:
+        out_lines = lines[:i+1]
+        out_lines.append("overlay[local?]\n")
+        out_lines.append("module;\n")
+        out_lines.extend(lines[i+1:])
+        return (f"Annotating \"{filename}\" which has a file-level module qldoc", out_lines)
+
+    out_lines = ["overlay[local?]\n", "module;\n", "\n"] + lines
+    return (f"Annotating \"{filename}\" without file-level module qldoc", out_lines)
+
+
+def insert_overlay_caller_annotations(lines):
+    '''
+    Mark pragma[inline] predicates as overlay[caller?] if they are not declared private.
+    '''
+    out_lines = []
+    for i, line in enumerate(lines):
+        trimmed = line.strip()
+        if trimmed == "pragma[inline]":
+            if i + 1 < len(lines) and not "private" in lines[i+1]:
+                whitespace = line[0: line.find(trimmed)]
+                out_lines.append(f"{whitespace}overlay[caller?]\n")
+        out_lines.append(line)
+    return out_lines
+
+
+def annotate_as_appropriate(filename, lines):
+    '''
+    Insert new overlay[...] annotations according to heuristics in files without existing
+    overlay annotations.
+
+    Returns None if no annotations are needed. Otherwise, returns a pair consisting of a
+    string describing the action taken and the modified content as a list of lines.
+    '''
+    if has_overlay_annotations(lines):
+        return None
+
+    # These simple heuristics filter out those .qll files that we no _not_ want to annotate
+    # as overlay[local?].  It is not clear that these heuristics are exactly what we want,
+    # but they seem to work well enough for now (as determined by speed and accuracy numbers).
+    if (filename.endswith("Test.qll") or
+        ((filename.endswith("Query.qll") or filename.endswith("Config.qll")) and
+         any("implements DataFlow::ConfigSig" in line for line in lines))):
+        return None
+    elif not any(line for line in lines if line.strip()):
+        return None
+
+    lines = insert_overlay_caller_annotations(lines)
+    return insert_toplevel_maybe_local_annotation(filename, lines)
+
+
+def process_single_file(write, filename):
+    '''
+    Process a single file, annotating it as appropriate.
+    If write is set, the changes are written back to the file.
+    Returns True if the file requires changes.
+    '''
+    with open(filename) as f:
+        old = [line for line in f]
+
+    annotate_result = annotate_as_appropriate(filename, old)
+    if annotate_result is None:
+        return False
+
+    if not write:
+        return True
+
+    new = annotate_result[1]
+
+    diff = context_diff(old, new, fromfile=filename, tofile=filename)
+    diff = [line for line in diff]
+    if diff:
+        print(annotate_result[0])
+        for line in diff:
+            print(line.rstrip())
+        with open(filename, "w") as out_file:
+            for line in new:
+                out_file.write(line)
+
+    return True
+
+
+if len(sys.argv) > 1 and sys.argv[1] == "--check":
+  check = True
+  langs = sys.argv[2:]
+else:
+  check = False
+  langs = sys.argv[1:]
+
+dirs = []
+for lang in langs:
+    if lang in ["cpp", "go", "csharp", "java", "javascript", "python", "ruby", "rust", "swift"]:
+        dirs.append(f"{lang}/ql/lib")
+    else:
+        raise Exception(f"Unknown language \"{lang}\".")
+
+if dirs:
+    dirs.append("shared")
+
+missingAnnotations = []
+
+for roots in dirs:
+    for dirpath, dirnames, filenames in os.walk(roots):
+        for filename in filenames:
+            if filename.endswith(".qll") and not dirpath.endswith("tutorial"):
+                path = os.path.join(dirpath, filename)
+                res = process_single_file(not check, path)
+                if check and res:
+                    missingAnnotations.append(path)
+
+
+if len(missingAnnotations) > 0:
+    print("The following files have no overlay annotations:")
+    for path in missingAnnotations[:10]:
+      print("- " + path)
+    if len(missingAnnotations) > 10:
+      print("and " + str(len(missingAnnotations) - 10) + " additional files.")
+    print()
+    print("Please manually add overlay annotations or use the config/add-overlay-annotations.py script to automatically add sensible default overlay annotations.")
+    exit(1)

config/dbscheme-fragments.json

@@ -1,16 +1,19 @@
 {
   "files": [
+    "cpp/ql/lib/semmlecode.cpp.dbscheme",
     "javascript/ql/lib/semmlecode.javascript.dbscheme",
     "python/ql/lib/semmlecode.python.dbscheme",
     "ruby/ql/lib/ruby.dbscheme",
     "ql/ql/src/ql.dbscheme"
   ],
   "fragments": [
+    "/*- Compilations -*/",
     "/*- External data -*/",
     "/*- Files and folders -*/",
     "/*- Diagnostic messages -*/",
     "/*- Diagnostic messages: severity -*/",
     "/*- Source location prefix -*/",
+    "/*- Database metadata -*/",
     "/*- Lines of code -*/",
     "/*- Configuration files with key value pairs -*/",
     "/*- YAML -*/",
@@ -20,6 +23,7 @@
     "/*- DEPRECATED: Snapshot date -*/",
     "/*- DEPRECATED: Duplicate code -*/",
     "/*- DEPRECATED: Version control data -*/",
+    "/*- C++ dbscheme -*/",
     "/*- JavaScript-specific part -*/",
     "/*- Ruby dbscheme -*/",
     "/*- Erb dbscheme -*/",
@@ -31,4 +35,4 @@
     "/*- Python dbscheme -*/",
     "/*- Empty location -*/"
   ]
-}
+}

config/opcode-qldoc.py

@@ -8,9 +8,9 @@ needs_an_re = re.compile(r'^(?!Unary)[AEIOU]')  # Name requiring "an" instead of
 start_qldoc_re = re.compile(r'^\s*/\*\*')  # Start of a QLDoc comment
 end_qldoc_re = re.compile(r'\*/\s*$')  # End of a QLDoc comment
 blank_qldoc_line_re = re.compile(r'^\s*\*\s*$')  # A line in a QLDoc comment with only the '*'
-instruction_class_re = re.compile(r'^class (?P<name>[A-aa-z0-9]+)Instruction\s')  # Declaration of an `Instruction` class
-opcode_base_class_re = re.compile(r'^abstract class (?P<name>[A-aa-z0-9]+)Opcode\s')  # Declaration of an `Opcode` base class
-opcode_class_re = re.compile(r'^  class (?P<name>[A-aa-z0-9]+)\s')  # Declaration of an `Opcode` class
+instruction_class_re = re.compile(r'^class (?P<name>[A-Za-z0-9]+)Instruction\s')  # Declaration of an `Instruction` class
+opcode_base_class_re = re.compile(r'^abstract class (?P<name>[A-Za-z0-9]+)Opcode\s')  # Declaration of an `Opcode` base class
+opcode_class_re = re.compile(r'^  class (?P<name>[A-Za-z0-9]+)\s')  # Declaration of an `Opcode` class
 
 script_dir = path.realpath(path.dirname(__file__))
 instruction_path = path.realpath(path.join(script_dir, '../cpp/ql/src/semmle/code/cpp/ir/implementation/raw/Instruction.qll'))

cpp/bulk_generation_targets.yml

@@ -0,0 +1,31 @@
+language: cpp
+strategy: dca
+destination: cpp/ql/lib/ext/generated
+targets:
+- name: zlib
+  with-sinks: false
+  with-sources: false
+- name: brotli
+  with-sinks: false
+  with-sources: false
+- name: libidn2
+  with-sinks: false
+  with-sources: false
+- name: libssh2
+  with-sinks: false
+  with-sources: false
+- name: sqlite
+  with-sinks: false
+  with-sources: false
+- name: openssl
+  with-sinks: false
+  with-sources: false
+- name: nghttp2
+  with-sinks: false
+  with-sources: false
+- name: libuv
+  with-sinks: false
+  with-sources: false
+- name: curl
+  with-sinks: false
+  with-sources: false

cpp/downgrades/3c45f8b9e71ec723bf50c40581e1f18f4f25e290/lambdas.ql

@@ -0,0 +1,7 @@
+class LambdaExpr extends @lambdaexpr {
+  string toString() { none() }
+}
+
+from LambdaExpr lambda, string default_capture, boolean has_explicit_return_type
+where lambdas(lambda, default_capture, has_explicit_return_type, _)
+select lambda, default_capture, has_explicit_return_type

cpp/downgrades/3c45f8b9e71ec723bf50c40581e1f18f4f25e290/upgrade.properties

@@ -0,0 +1,3 @@
+description: capture whether a lambda has an explicitly specified parameter list.
+compatibility: full
+lambdas.rel: run lambdas.qlo

cpp/downgrades/5340d6d5f428557632b1a50113e406430f29ef7d/upgrade.properties

@@ -0,0 +1,2 @@
+description: Uncomment cases in dbscheme
+compatibility: full

cpp/downgrades/5491582ac8511726e12fae3e2399000f9201cd9a/upgrade.properties

@@ -0,0 +1,2 @@
+description: Remove unused external_package tables from the dbscheme
+compatibility: full

cpp/downgrades/59cb96ca699929b63941e81905f9b8de7eed59a6/preprocdirects.ql

@@ -11,7 +11,7 @@ int getKind(int kind) {
   if kind = 14
   then result = 6 // Represent MSFT #import as #include
   else
-    if kind = 15 or kind = 6
+    if kind = 15 or kind = 16
     then result = 3 // Represent #elifdef and #elifndef as #elif
     else result = kind
 }

cpp/downgrades/7bc12b02a4363149f0727a4bce07952dbb9d98aa/builtintypes.ql

@@ -0,0 +1,14 @@
+class BuiltinType extends @builtintype {
+  string toString() { none() }
+}
+
+from BuiltinType type, string name, int kind, int kind_new, int size, int sign, int alignment
+where
+  builtintypes(type, name, kind, size, sign, alignment) and
+  if
+    type instanceof @complex_fp16 or
+    type instanceof @complex_std_bfloat16 or
+    type instanceof @complex_std_float16
+  then kind_new = 2
+  else kind_new = kind
+select type, name, kind_new, size, sign, alignment

cpp/downgrades/7bc12b02a4363149f0727a4bce07952dbb9d98aa/upgrade.properties

@@ -0,0 +1,3 @@
+description: Introduce new complex 16-bit floating-point types
+compatibility: backwards
+builtintypes.rel: run builtintypes.qlo

cpp/downgrades/801b2f03360d78c85f51fbad9b75956fa8d58b00/functions.ql

@@ -0,0 +1,9 @@
+class Function extends @function {
+  string toString() { none() }
+}
+
+from Function f, string n, int k, int new_k
+where
+  functions(f, n, k) and
+  if builtin_functions(f) then new_k = 6 else new_k = k
+select f, n, new_k

cpp/downgrades/801b2f03360d78c85f51fbad9b75956fa8d58b00/upgrade.properties

@@ -0,0 +1,4 @@
+description: Move builtin function identification to its own table
+compatibility: full
+functions.rel: run functions.qlo
+builtin_functions.rel: delete

cpp/downgrades/827dbc206ea55377e032a8a934c8903fedc50fa0/upgrade.properties

@@ -0,0 +1,2 @@
+description: sync dbscheme and delete svn tables
+compatibility: full

cpp/downgrades/9baef67d1ffc1551429dbe1c1130815693e28218/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add a predicate `getAnAttribute` to `Namespace`
+compatibility: full
+namespaceattributes.rel: delete