API graphs: Prune nodes before computing trackUseNode

.bazelrc

@@ -1,3 +0,0 @@
-build --repo_env=CC=clang --repo_env=CXX=clang++ --cxxopt="-std=c++17"
-
-try-import %workspace%/local.bazelrc

.bazelversion

@@ -1 +0,0 @@
-5.0.0

.codeqlmanifest.json

@@ -0,0 +1,10 @@
+{
+    "provide": [
+        "ql/lib/qlpack.yml",
+        "ql/src/qlpack.yml",
+        "ql/consistency-queries/qlpack.yml",
+        "ql/test/qlpack.yml",
+        "ql/examples/qlpack.yml",
+        "extractor-pack/codeql-extractor.yml"
+    ]
+}

.devcontainer/Dockerfile

@@ -0,0 +1,15 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.162.0/containers/rust/.devcontainer/base.Dockerfile
+
+FROM mcr.microsoft.com/vscode/devcontainers/rust:0-1
+
+RUN apt-key --keyring /usr/share/keyrings/githubcli-archive-keyring.gpg adv \
+    --keyserver keyserver.ubuntu.com --recv-key C99B11DEB97541F0 && \
+    echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages $(lsb_release -cs) main" \
+    |  tee /etc/apt/sources.list.d/github-cli2.list > /dev/null
+
+
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install --no-install-recommends gh 
+
+COPY post_create.sh /bin/post_create.sh
+COPY post_attach.sh /bin/post_attach.sh

.devcontainer/devcontainer.json

@@ -1,16 +1,39 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.162.0/containers/rust
 {
-  "extensions": [
-    "rust-lang.rust",
-    "bungcip.better-toml",
-    "github.vscode-codeql",
-    "hbenl.vscode-test-explorer",
-    "ms-vscode.test-adapter-converter",
-    "slevesque.vscode-zipexplorer"
-  ],
-  "settings": {
-    "files.watcherExclude": {
-      "**/target/**": true
-    },
-    "codeQL.runningQueries.memory": 2048
-  }
-}
+	"name": "Rust",
+	"build": {
+		"dockerfile": "Dockerfile"
+	},
+	"runArgs": [
+		"--cap-add=SYS_PTRACE",
+		"--security-opt",
+		"seccomp=unconfined"
+	],
+	// Set *default* container specific settings.json values on container create.
+	"settings": {
+		"terminal.integrated.shell.linux": "/bin/bash",
+		"lldb.executable": "/usr/bin/lldb",
+		// VS Code don't watch files under ./target
+		"files.watcherExclude": {
+			"**/target/**": true
+		}
+	},
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"rust-lang.rust",
+		"bungcip.better-toml",
+		"vadimcn.vscode-lldb",
+		"mutantdino.resourcemonitor",
+		"ms-azuretools.vscode-docker",
+		"github.vscode-codeql"
+	],
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "rustc --version",
+	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode",
+	"postCreateCommand": [ "/bin/post_create.sh" ],
+	"postAttachCommand": [ "flock", "-E", "0", "-n", "/var/lock/post_attach.lock", "/bin/post_attach.sh" ]
+}

.devcontainer/post_attach.sh

@@ -0,0 +1,37 @@
+#! /bin/bash
+set -xe
+
+echo "Check installed CodeQL version"
+CURRENT_CODEQL_BIN=$(readlink -e /usr/local/bin/codeql || echo "")
+LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
+
+BASE_DIR=/home/vscode/codeql-binaries
+mkdir -p "${BASE_DIR}"
+LATEST_CODEQL_DIR="${BASE_DIR}/codeql-${LATEST}"
+LATEST_CODEQL_BIN="${LATEST_CODEQL_DIR}/codeql/codeql"
+
+if [ "${CURRENT_CODEQL_BIN}" != "${LATEST_CODEQL_BIN}" ]; then
+  echo "Installing CodeQL ${LATEST}"
+  TMPDIR=$(mktemp -d -p "$(dirname ${LATEST_CODEQL_DIR})")
+  gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip -D "${TMPDIR}" "$LATEST" 
+  unzip -oq "${TMPDIR}/codeql-linux64.zip" -d "${TMPDIR}"
+  rm -f "${TMPDIR}/codeql-linux64.zip"
+  mv "${TMPDIR}" "${LATEST_CODEQL_DIR}"
+  test -x "${LATEST_CODEQL_BIN}" && sudo ln -sf "${LATEST_CODEQL_BIN}" /usr/local/bin/codeql
+  if [[ "${CURRENT_CODEQL_BIN}" =~ .*/codeql/codeql ]]; then
+    rm -rf "$(dirname $(dirname ${CURRENT_CODEQL_BIN}))"
+  fi
+fi
+
+echo "Build the Ruby extractor"
+
+# clone the git dependencies using "git clone" because cargo's builtin git support is rather slow
+REPO_DIR="${CARGO_HOME:-/home/vscode/.cargo}/git/db" 
+REPO_DIR_ERB="${REPO_DIR}/tree-sitter-embedded-template-4c796e3340c233b6"
+REPO_DIR_RUBY="${REPO_DIR}/tree-sitter-ruby-666a40ce046f8e7a"
+
+mkdir -p "${REPO_DIR}"
+test -e "${REPO_DIR_ERB}" || git clone -q --bare https://github.com/tree-sitter/tree-sitter-embedded-template "${REPO_DIR_ERB}"
+test -e "${REPO_DIR_RUBY}" || git clone -q --bare https://github.com/tree-sitter/tree-sitter-ruby.git "${REPO_DIR_RUBY}"
+
+scripts/create-extractor-pack.sh

.devcontainer/post_create.sh

@@ -0,0 +1,4 @@
+#! /bin/bash
+
+mkdir -p /home/vscode/.config/codeql
+echo '--search-path /workspaces/codeql-ruby' >> /home/vscode/.config/codeql/config

.devcontainer/swift/Dockerfile

@@ -1,9 +0,0 @@
-# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.236.0/containers/cpp/.devcontainer/base.Dockerfile
-
-# [Choice] Debian / Ubuntu version (use Debian 11, Ubuntu 18.04/22.04 on local arm64/Apple Silicon): debian-11, debian-10, ubuntu-22.04, ubuntu-20.04, ubuntu-18.04
-FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-ubuntu-22.04
-
-USER root
-ADD root.sh /tmp/root.sh
-ADD update-codeql.sh /usr/local/bin/update-codeql
-RUN bash /tmp/root.sh && rm /tmp/root.sh

.devcontainer/swift/devcontainer.json

@@ -1,25 +0,0 @@
-{
-	"extensions": [
-		"github.vscode-codeql",
-		"hbenl.vscode-test-explorer",
-		"ms-vscode.test-adapter-converter",
-		"slevesque.vscode-zipexplorer",
-		"ms-vscode.cpptools"
-	],
-	"settings": {
-		"files.watcherExclude": {
-			"**/target/**": true
-		},
-		"codeQL.runningQueries.memory": 2048
-	},
-	"build": {
-		"dockerfile": "Dockerfile",
-	},
-	"runArgs": [
-		"--cap-add=SYS_PTRACE",
-		"--security-opt",
-		"seccomp=unconfined"
-	],
-	"remoteUser": "vscode",
-	"onCreateCommand": ".devcontainer/swift/user.sh"
-}

.devcontainer/swift/root.sh

@@ -1,22 +0,0 @@
-set -xe
-
-BAZELISK_VERSION=v1.12.0
-BAZELISK_DOWNLOAD_SHA=6b0bcb2ea15bca16fffabe6fda75803440375354c085480fe361d2cbf32501db
-
-apt-get update
-export DEBIAN_FRONTEND=noninteractive
-apt-get -y install --no-install-recommends \
-    zlib1g-dev \
-    uuid-dev \
-    python3-distutils \
-    python3-pip \
-    bash-completion
-
-# Install Bazel
-curl -fSsL -o /usr/local/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-amd64
-echo "${BAZELISK_DOWNLOAD_SHA} */usr/local/bin/bazelisk" | sha256sum --check -
-chmod 0755 /usr/local/bin/bazelisk
-ln -s bazelisk /usr/local/bin/bazel
-
-# install latest codeql
-update-codeql

.devcontainer/swift/update-codeql.sh

@@ -1,20 +0,0 @@
-#!/bin/bash -e
-
-URL=https://github.com/github/codeql-cli-binaries/releases
-LATEST_VERSION=$(curl -L -s -H 'Accept: application/json' $URL/latest | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/')
-CURRENT_VERSION=v$(codeql version 2>/dev/null | sed -ne 's/.*release \([0-9.]*\)\./\1/p')
-if [[ $CURRENT_VERSION != $LATEST_VERSION ]]; then
-  if [[ $UID != 0 ]]; then
-    echo "update required, please run this script with sudo:"
-    echo "  sudo $0"
-    exit 1
-  fi
-  ZIP=$(mktemp codeql.XXXX.zip)
-  curl -fSqL -o $ZIP $URL/download/$LATEST_VERSION/codeql-linux64.zip
-  unzip -q $ZIP -d /opt
-  rm $ZIP
-  ln -sf /opt/codeql/codeql /usr/local/bin/codeql
-  echo installed version $LATEST_VERSION
-else
-  echo current version $CURRENT_VERSION is up-to-date
-fi

.devcontainer/swift/user.sh

@@ -1,13 +0,0 @@
-set -xe
-
-# add the workspace to the codeql search path
-mkdir -p /home/vscode/.config/codeql
-echo "--search-path /workspaces/codeql" > /home/vscode/.config/codeql/config
-
-# create a swift extractor pack with the current state
-cd /workspaces/codeql
-bazel run swift/create-extractor-pack
-
-#install and set up pre-commit
-python3 -m pip install pre-commit --no-warn-script-location
-$HOME/.local/bin/pre-commit install

.editorconfig

@@ -1,2 +0,0 @@
-[*]
-end_of_line = lf

.gitattributes

@@ -1,73 +1 @@
-# Text files will be normalized to LF line endings in the Git database, and will keep those LF line
-# endings in the working tree even on Windows. If you make changes below, you should renormalize the
-# affected files by running the following from the root of this repo (requires Git 2.16 or greater):
-#
-# git add --renormalize .
-# git status  [just to show what files were renormalized]
-# git commit -m "Normalize line endings"
-
-# Anything Git auto-detects as text gets normalized and checked out as LF
-* text=auto eol=lf
-
-# Explicitly set a bunch of known extensions to text, in case auto detection gets confused.
-*.ql text
-*.qll text
-*.qlref text
-*.dbscheme text
-*.qhelp text
-*.html text
-*.htm text
-*.xhtml text
-*.xhtm text
-*.js text
-*.mjs text
-*.ts text
-*.json text
-*.yml text
-*.yaml text
-*.c text
-*.cpp text
-*.h text
-*.hpp text
-*.md text
-*.stats text
-*.xml text
-*.sh text
-*.pl text
-*.java text
-*.cs text
-*.py text
-*.lua text
-*.expected text
-*.go text
-
-# Explicitly set a bunch of known extensions to binary, because Git < 2.10 will treat
-# `* text=auto eol=lf` as `* text eol=lf`
-*.png -text
-*.jpg -text
-*.jpeg -text
-*.gif -text
-*.dll -text
-*.pdb -text
-
-java/ql/test/stubs/**/*.java linguist-generated=true
-java/ql/test/experimental/stubs/**/*.java linguist-generated=true
-
-# Force git not to modify line endings for go or html files under the go/ql directory
-go/ql/**/*.go -text
-go/ql/**/*.html -text
-# Force git not to modify line endings for go dbschemes
-go/*.dbscheme -text
-# Preserve unusual line ending from codeql-go merge
-go/extractor/opencsv/CSVReader.java -text
-
-# For some languages, upgrade script testing references really old dbscheme
-# files from legacy upgrades that have CRLF line endings. Since upgrade
-# resolution relies on object hashes, we must suppress line ending conversion
-# for those testing dbscheme files.
-*/ql/lib/upgrades/initial/*.dbscheme -text
-
-# Generated test files - these are synced from the standard JavaScript libraries using
-# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge
+Cargo.lock   -diff -whitespace

.github/ISSUE_TEMPLATE/ql---general.md

@@ -1,14 +0,0 @@
----
-name: General issue
-about: Tell us if you think something is wrong or if you have a question
-title: General issue
-labels: question
-assignees: ''
-
----
-
-**Description of the issue**
-
-<!-- Please explain briefly what is the problem.
-If it is about a GitHub project, please include its URL. -->
-

.github/ISSUE_TEMPLATE/ql--false-positive.md

@@ -1,36 +0,0 @@
----
-name: CodeQL false positive
-about: Report CodeQL alerts that you think should not have been detected (not applicable, not exploitable, etc.)
-title: False positive
-labels: false-positive
-assignees: ''
-
----
-
-**Description of the false positive**
-
-<!-- Please explain briefly why you think it shouldn't be included. -->
-
-**Code samples or links to source code**
-
-<!--
-For open source code: file links with line numbers on GitHub, for example:
-https://github.com/github/codeql/blob/dc440aaee6695deb0d9676b87e06ea984e1b4ae5/javascript/ql/test/query-tests/Security/CWE-078/CommandInjection/exec-sh2.js#L10
-
-For closed source code: (redacted) code samples that illustrate the problem, for example:
-
-```
-function execSh(command, options) {
-    return cp.spawn(getShell(), ["-c", command], options) // <- command line injection
-};
-```
--->
-
-**URL to the alert on GitHub code scanning (optional)**
-
-<!--
-1. Open the project on GitHub.com.
-2. Switch to the `Security` tab.
-3. Browse to the alert that you would like to report.
-4. Copy and paste the page URL here.
--->

.github/actions/cache-query-compilation/action.yml

@@ -1,54 +0,0 @@
-name: Cache query compilation
-description: Caches CodeQL compilation caches - should be run both on PRs and pushes to main.
-
-inputs:
-  key:
-    description: 'The cache key to use - should be unique to the workflow'
-    required: true
-
-outputs:
-  cache-dir:
-    description: "The directory where the cache was stored"
-    value: ${{ steps.fill-compilation-dir.outputs.compdir }}
-
-runs:
-  using: composite
-  steps:
-    # calculate the merge-base with main, in a way that works both on PRs and pushes to main.
-    - name: Calculate merge-base
-      shell: bash
-      if: ${{ github.event_name == 'pull_request' }}
-      env:
-        BASE_BRANCH: ${{ github.base_ref }}
-      run: |
-        MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
-        echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV
-    - name: Restore cache (PR)
-      if: ${{ github.event_name == 'pull_request' }}
-      uses: actions/cache/restore@v3
-      with:
-        path: '**/.cache'
-        key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }}
-        restore-keys: |
-          codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
-          codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-
-          codeql-compile-${{ inputs.key }}-main-
-    - name: Fill cache (only branch push)
-      if: ${{ github.event_name != 'pull_request' }}
-      uses: actions/cache@v3
-      with:
-        path: '**/.cache'
-        key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
-        restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
-          codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
-          codeql-compile-${{ inputs.key }}-main-
-    - name: Fill compilation cache directory
-      id: fill-compilation-dir
-      shell: bash
-      run: |
-        # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-        node $GITHUB_WORKSPACE/.github/actions/cache-query-compilation/move-caches.js ${COMBINED_CACHE_DIR}
-
-        echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
-      env:
-        COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir

.github/actions/cache-query-compilation/move-caches.js

@@ -1,75 +0,0 @@
-// # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-// mkdir -p ${COMBINED_CACHE_DIR}
-// rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-// # copy the contents of the .cache folders into the combined cache folder.
-// cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-// # clean up the .cache folders
-// rm -rf **/.cache/*
-
-const fs = require("fs");
-const path = require("path");
-
-// the first argv is the cache folder to create.
-const COMBINED_CACHE_DIR = process.argv[2];
-
-function* walkCaches(dir) {
-  const files = fs.readdirSync(dir, { withFileTypes: true });
-  for (const file of files) {
-    if (file.isDirectory()) {
-      const filePath = path.join(dir, file.name);
-      yield* walkCaches(filePath);
-      if (file.name === ".cache") {
-        yield filePath;
-      }
-    }
-  }
-}
-
-async function copyDir(src, dest) {
-  for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
-    const srcPath = path.join(src, file.name);
-    const destPath = path.join(dest, file.name);
-    if (file.isDirectory()) {
-      if (!fs.existsSync(destPath)) {
-        fs.mkdirSync(destPath);
-      }
-      await copyDir(srcPath, destPath);
-    } else {
-      await fs.promises.copyFile(srcPath, destPath);
-    }
-  }
-}
-
-async function main() {
-  const cacheDirs = [...walkCaches(".")];
-
-  for (const dir of cacheDirs) {
-    console.log(`Found .cache dir at ${dir}`);
-  }
-
-  // mkdir -p ${COMBINED_CACHE_DIR}
-  fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
-
-  // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-  await Promise.all(
-    cacheDirs.map((cacheDir) =>
-      (async function () {
-        await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
-        await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
-      })()
-    )
-  );
-
-  // # copy the contents of the .cache folders into the combined cache folder.
-  // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-  await Promise.all(
-    cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
-  );
-
-  // # clean up the .cache folders
-  // rm -rf **/.cache/*
-  await Promise.all(
-    cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
-  );
-}
-main();

.github/actions/create-extractor-pack/action.yml

@@ -0,0 +1,15 @@
+name: Build Ruby CodeQL pack
+description: Builds the Ruby CodeQL pack
+runs:
+  using: composite
+  steps:
+    - uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          target
+        key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
+    - name: Build Extractor
+      shell: bash
+      run: env "PATH=$PATH:${{ github.workspace }}/codeql" scripts/create-extractor-pack.sh

.github/actions/fetch-codeql/action.yml

@@ -1,24 +1,13 @@
 name: Fetch CodeQL
 description: Fetches the latest version of CodeQL
-
-inputs:
-  channel:
-    description: 'The CodeQL channel to use'
-    required: false
-    default: 'nightly'
-
 runs:
   using: composite
   steps:
     - name: Fetch CodeQL
       shell: bash
+      run: |
+        LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
+        gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
+        unzip -q codeql-linux64.zip
       env:
         GITHUB_TOKEN: ${{ github.token }}
-        CHANNEL: ${{ inputs.channel }}
-      run: |
-        gh extension install github/gh-codeql
-        gh codeql set-channel "$CHANNEL"
-        gh codeql version
-        printf "CODEQL_FETCHED_CODEQL_PATH=" >> "${GITHUB_ENV}"
-        gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_ENV}"
-        gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_PATH}"

.github/actions/find-latest-bundle/action.yml

@@ -1,26 +0,0 @@
-name: Find Latest CodeQL Bundle
-description: Finds the URL of the latest released version of the CodeQL bundle.
-outputs:
-  url:
-    description: The download URL of the latest CodeQL bundle release
-    value: ${{ steps.find-latest.outputs.url }}
-runs:
-  using: composite
-  steps:
-    - name: Find Latest Release
-      id: find-latest
-      shell: pwsh
-      run: |
-        $Latest = gh release list --repo github/codeql-action --exclude-drafts --limit 1000 |
-          ForEach-Object { $C = $_ -split "`t"; return @{ type = $C[1]; tag = $C[2]; } } |
-          Where-Object { $_.type -eq 'Latest' }
-
-        $Tag = $Latest.tag
-        if ($Tag -eq '') {
-          throw 'Failed to find latest bundle release.'
-        }
-
-        Write-Output "Latest bundle tag is '${Tag}'."
-        "url=https://github.com/github/codeql-action/releases/download/${Tag}/codeql-bundle-linux64.tar.gz" >> $env:GITHUB_OUTPUT
-      env:
-        GITHUB_TOKEN: ${{ github.token }}

.github/actions/os-version/action.yml

@@ -1,32 +0,0 @@
-name: OS Version
-description: Get OS version.
-
-outputs:
-  version:
-    description: "OS version"
-    value: ${{ steps.version.outputs.version }}
-
-runs:
-  using: composite
-  steps:
-    - if: runner.os == 'Linux'
-      shell: bash
-      run: |
-        . /etc/os-release
-        echo "VERSION=${NAME} ${VERSION}" >> $GITHUB_ENV
-    - if: runner.os == 'Windows'
-      shell: powershell
-      run: |
-        $objects = systeminfo.exe /FO CSV | ConvertFrom-Csv
-        "VERSION=$($objects.'OS Name') $($objects.'OS Version')" >> $env:GITHUB_ENV
-    - if: runner.os == 'macOS'
-      shell: bash
-      run: |
-        echo "VERSION=$(sw_vers -productName) $(sw_vers -productVersion)" >> $GITHUB_ENV
-    - name: Emit OS version
-      id: version
-      shell: bash
-      run: |
-        echo "$VERSION"
-        echo "version=${VERSION}" >> $GITHUB_OUTPUT
-

.github/codeql/codeql-config.yml

@@ -1,11 +0,0 @@
-name: "CodeQL config"
-
-queries:
-  - uses: security-and-quality
-
-paths-ignore:
-  - '/cpp/'
-  - '/java/'
-  - '/python/'
-  - '/javascript/ql/test'
-  - '/javascript/extractor/tests'

.github/dependabot.yml

@@ -1,14 +1,18 @@
 version: 2
 updates:
   - package-ecosystem: "cargo"
-    directory: "ruby"
+    directory: "/node-types"
     schedule:
       interval: "daily"
-
-  - package-ecosystem: "github-actions"
-    directory: "/"
+  - package-ecosystem: "cargo"
+    directory: "/generator"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "cargo"
+    directory: "/extractor"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "cargo"
+    directory: "/autobuilder"
     schedule:
       interval: "daily"
-    ignore:
-      - dependency-name: '*'
-        update-types: ['version-update:semver-patch', 'version-update:semver-minor']

.github/labeler.yml

@@ -1,56 +0,0 @@
-"C++":
-  - cpp/**/*
-  - change-notes/**/*cpp*
-
-"C#":
-  - csharp/**/*
-  - change-notes/**/*csharp*
-
-Go:
-  - go/**/*
-  - change-notes/**/*go.*
-
-Java:
-  - any: [ 'java/**/*', '!java/kotlin-extractor/**/*', '!java/kotlin-explorer/**/*', '!java/ql/test/kotlin/**/*' ]
-  - change-notes/**/*java.*
-
-JS:
-  - any: [ 'javascript/**/*', '!javascript/ql/experimental/adaptivethreatmodeling/**/*' ]
-  - change-notes/**/*javascript*
-
-Kotlin:
-  - java/kotlin-extractor/**/*
-  - java/kotlin-explorer/**/*
-  - java/ql/test/kotlin/**/*
-
-Python:
-  - python/**/*
-  - change-notes/**/*python*
-
-Ruby:
-  - ruby/**/*
-  - change-notes/**/*ruby*
-
-Swift:
-  - swift/**/*
-  - change-notes/**/*swift*
-
-documentation:
-  - "**/*.qhelp"
-  - "**/*.md"
-  - docs/**/*
-
-"QL-for-QL":
-  - ql/**/*
-  - .github/workflows/ql-for-ql*
-
-# Since these are all shared files that need to be synced, just pick _one_ copy of each.
-"DataFlow Library":
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"
-
-"ATM":
-  - javascript/ql/experimental/adaptivethreatmodeling/**/*

.github/problem-matchers/codeql-query-format.json

@@ -1,14 +0,0 @@
-{
-    "problemMatcher": [
-        {
-            "owner": "codeql-query-format",
-            "pattern": [
-                {
-                    "regexp": "^((.*) would change by autoformatting\\.)$",
-                    "file": 2,
-                    "message": 1
-                }
-            ]
-        }
-    ]
-}

.github/problem-matchers/codeql-syntax-check.json

@@ -1,17 +0,0 @@
-{
-    "problemMatcher": [
-        {
-            "owner": "codeql-syntax-check",
-            "pattern": [
-                {
-                    "regexp": "^((ERROR|WARNING): .* \\((.*):(\\d+),(\\d+)-\\d+\\))$",
-                    "message": 1,
-                    "file": 3,
-                    "line": 4,
-                    "col": 5,
-                    "severity": 2
-                }
-            ]
-        }
-    ]
-}

.github/problem-matchers/codeql-test-run.json

@@ -1,14 +0,0 @@
-{
-    "problemMatcher": [
-        {
-            "owner": "codeql-test-run",
-            "pattern": [
-                {
-                    "regexp": "(\\[.*\\] FAILED\\((RESULT|COMPILATION)\\) (.*))$",
-                    "file": 3,
-                    "message": 1
-                }
-            ]
-        }
-    ]
-}

.github/problem-matchers/make.json

@@ -1,13 +0,0 @@
-{
-    "problemMatcher": [
-        {
-            "owner": "make",
-            "pattern": [
-                {
-                    "regexp": "^(make: \\*\\*\\* .*)$",
-                    "message": 1
-                }
-            ]
-        }
-    ]
-}

.github/workflows/atm-check-query-suite.yml

@@ -1,102 +0,0 @@
-name: "ATM - Check query suite"
-
-env:
-  QUERY_PACK: javascript/ql/experimental/adaptivethreatmodeling/src
-  QUERY_SUITE: codeql-suites/javascript-atm-code-scanning.qls
-
-on:
-  pull_request:
-    paths:
-      - ".github/workflows/atm-check-query-suite.yml"
-      - "javascript/ql/experimental/adaptivethreatmodeling/**"
-  workflow_dispatch:
-
-jobs:
-  atm-check-query-suite:
-    runs-on: ubuntu-latest-xl
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Setup CodeQL
-        uses: ./.github/actions/fetch-codeql
-        with:
-          channel: release
-
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: atm-suite
-
-      - name: Install ATM model
-        run: |
-          set -exu
-
-          # Install dependencies of ATM query pack, i.e. the ATM model
-          codeql pack install "${QUERY_PACK}"
-
-          # Retrieve model checksum
-          model_checksum=$(codeql resolve extensions "${QUERY_PACK}/${QUERY_SUITE}" | jq -r '.models[0].checksum')
-
-          # Trust the model so that we can use it in the ATM boosted queries
-          mkdir -p "$HOME/.config/codeql"
-          echo "--insecurely-execute-ml-model-checksums ${model_checksum}" >> "$HOME/.config/codeql/config"
-
-      - name: Create test DB
-        run: |
-          DB_PATH="${RUNNER_TEMP}/db"
-          echo "DB_PATH=${DB_PATH}" >> "${GITHUB_ENV}"
-
-          codeql database create "${DB_PATH}" --source-root config/atm --language javascript 
-
-      - name: Run ATM query suite
-        run: |
-          SARIF_PATH="${RUNNER_TEMP}/sarif.json"
-          echo "SARIF_PATH=${SARIF_PATH}" >> "${GITHUB_ENV}"
-
-          codeql database analyze \
-            --threads=0 \
-            --ram 50000 \
-            --format sarif-latest \
-            --output "${SARIF_PATH}" \
-            --sarif-group-rules-by-pack \
-            -vv \
-            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
-            -- \
-            "${DB_PATH}" \
-            "${QUERY_PACK}/${QUERY_SUITE}"
-
-      - name: Upload SARIF
-        uses: actions/upload-artifact@v3
-        with:
-          name: javascript-ml-powered-queries.sarif
-          path: "${{ env.SARIF_PATH }}"
-          retention-days: 5
-
-      - name: Check results
-        run: |
-          # We should run at least the ML-powered queries in `expected_rules`.
-          expected_rules="js/ml-powered/nosql-injection js/ml-powered/path-injection js/ml-powered/sql-injection js/ml-powered/xss"
-
-          for rule in ${expected_rules}; do
-            found_rule=$(jq --arg rule "${rule}" '[.runs[0].tool.extensions[].rules | select(. != null) |
-              flatten | .[].id] | any(. == $rule)' "${SARIF_PATH}")
-            if [[ "${found_rule}" != "true" ]]; then
-              echo "Expected SARIF output to contain rule '${rule}', but found no such rule."
-              exit 1
-            else
-              echo "Found rule '${rule}'."
-            fi
-          done
-
-          # We should have at least one alert from an ML-powered query.
-          num_alerts=$(jq '[.runs[0].results[] |
-            select(.properties.score != null and (.rule.id | startswith("js/ml-powered/")))] | length' \
-            "${SARIF_PATH}")
-          if [[ "${num_alerts}" -eq 0 ]]; then
-            echo "Expected to find at least one alert from an ML-powered query but found ${num_alerts}."
-            exit 1
-          else
-            echo "Found ${num_alerts} alerts from ML-powered queries.";
-          fi

.github/workflows/atm-model-integration-tests.yml

@@ -1,12 +0,0 @@
-name: ATM Model Integration Tests
-
-on:
-  workflow_dispatch:
-
-jobs:
-  hello-world:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: foo
-        run: echo "Hello world"

.github/workflows/build.yml

@@ -0,0 +1,229 @@
+name: Build / Release
+
+on:
+  push:
+    branches:
+      - main
+      - 'rc/*'
+  pull_request:
+    branches:
+      - main
+      - 'rc/*'
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Version tag to create"
+        required: false
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install GNU tar
+        if: runner.os == 'macOS'
+        run: |
+          brew install gnu-tar
+          echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('**/Cargo.lock') }}
+      - name: Check formatting
+        run: cargo fmt --all -- --check
+      - name: Build
+        run: cargo build --verbose
+      - name: Run tests
+        run: cargo test --verbose
+      - name: Release build
+        run: cargo build --release
+      - name: Generate dbscheme
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: target/release/ruby-generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+      - uses: actions/upload-artifact@v2
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        with:
+          name: ruby.dbscheme
+          path: ql/lib/ruby.dbscheme
+      - uses: actions/upload-artifact@v2
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        with:
+          name: TreeSitter.qll
+          path: ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+      - uses: actions/upload-artifact@v2
+        with:
+          name: extractor-${{ matrix.os }}
+          path: |
+            target/release/ruby-autobuilder
+            target/release/ruby-autobuilder.exe
+            target/release/ruby-extractor
+            target/release/ruby-extractor.exe
+          retention-days: 1
+  compile-queries:
+    runs-on: ubuntu-latest
+    env:
+      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
+    steps:
+      - uses: actions/checkout@v2
+      - name: Fetch CodeQL
+        run: |
+          LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
+          gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
+          unzip -q codeql-linux64.zip
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - name: Build Query Pack
+        run: |
+          codeql/codeql pack create ql/lib --output target/packs
+          codeql/codeql pack install ql/src
+          codeql/codeql pack create ql/src --output target/packs
+          PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
+          codeql/codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
+          (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
+      - name: Compile with previous CodeQL versions
+        run: |
+          for version in  $(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | sort --version-sort | tail -3 | head -2); do
+            rm -f codeql-linux64.zip
+            gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$version"
+            rm -rf codeql; unzip -q codeql-linux64.zip
+            codeql/codeql query compile target/packs/*
+          done
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: codeql-ruby-queries
+          path: |
+            target/packs/*
+          retention-days: 1
+
+  package:
+    runs-on: ubuntu-latest
+    needs: [build, compile-queries]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+        with:
+          name: ruby.dbscheme
+          path: ruby
+      - uses: actions/download-artifact@v2
+        with:
+          name: extractor-ubuntu-latest
+          path: linux64
+      - uses: actions/download-artifact@v2
+        with:
+          name: extractor-windows-latest
+          path: win64
+      - uses: actions/download-artifact@v2
+        with:
+          name: extractor-macos-latest
+          path: osx64
+      - run: |
+          mkdir -p ruby
+          cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
+          mkdir -p ruby/tools/{linux64,osx64,win64}
+          cp linux64/ruby-autobuilder ruby/tools/linux64/autobuilder
+          cp osx64/ruby-autobuilder ruby/tools/osx64/autobuilder
+          cp win64/ruby-autobuilder.exe ruby/tools/win64/autobuilder.exe
+          cp linux64/ruby-extractor ruby/tools/linux64/extractor
+          cp osx64/ruby-extractor ruby/tools/osx64/extractor
+          cp win64/ruby-extractor.exe ruby/tools/win64/extractor.exe
+          chmod +x ruby/tools/{linux64,osx64}/{autobuilder,extractor}
+          zip -rq codeql-ruby.zip ruby
+      - uses: actions/upload-artifact@v2
+        with:
+          name: codeql-ruby-pack
+          path: codeql-ruby.zip
+          retention-days: 1
+      - uses: actions/download-artifact@v2
+        with:
+          name: codeql-ruby-queries
+          path: qlpacks
+      - run: |
+          echo '{
+            "provide": [
+            "ruby/codeql-extractor.yml",
+            "qlpacks/*/*/*/qlpack.yml"
+            ]
+          }' > .codeqlmanifest.json
+          zip -rq codeql-ruby-bundle.zip .codeqlmanifest.json ruby qlpacks
+      - uses: actions/upload-artifact@v2
+        with:
+          name: codeql-ruby-bundle
+          path: codeql-ruby-bundle.zip
+          retention-days: 1
+      - if: ${{ github.event.inputs.tag }}
+        run: |
+          gh release create --prerelease \
+             --title "CodeQL Ruby (${{ github.event.inputs.tag }})" \
+             --target "${{ github.sha }}" \
+             "${{ github.event.inputs.tag }}" \
+             codeql-ruby-bundle.zip
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    runs-on: ${{ matrix.os }}
+    needs: [package]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          repository: Shopify/example-ruby-app
+          ref: 67a0decc5eb550f3a9228eda53925c3afd40dfe9
+      - name: Fetch CodeQL
+        shell: bash
+        run: |
+          LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
+          gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql.zip "$LATEST"
+          unzip -q codeql.zip
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        working-directory: ${{ runner.temp }}
+      - name: Download Ruby bundle
+        uses: actions/download-artifact@v2
+        with:
+          name: codeql-ruby-bundle
+          path: ${{ runner.temp }}
+      - name: Unzip Ruby bundle
+        shell: bash
+        run: unzip -q -d "${{ runner.temp }}/ruby-bundle" "${{ runner.temp }}/codeql-ruby-bundle.zip"
+      - name: Prepare test files
+        shell: bash
+        run: |
+          echo "import ruby select count(File f)" > "test.ql"
+          echo "| 4 |" > "test.expected"
+          echo 'name: sample-tests
+          version: 0.0.0
+          dependencies:
+            codeql/ruby-all: 0.0.1
+          extractor: ruby
+          tests: .
+          ' > qlpack.yml
+      - name: Run QL test
+        shell: bash
+        run: |
+          "${{ runner.temp }}/codeql/codeql" test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
+      - name: Create database
+        shell: bash
+        run: |
+          "${{ runner.temp }}/codeql/codeql" database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
+      - name: Analyze database
+        shell: bash
+        run: |
+          "${{ runner.temp }}/codeql/codeql" database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/check-change-note.yml

@@ -1,34 +0,0 @@
-name: Check change note
-
-on:
-  pull_request_target:
-    types: [labeled, unlabeled, opened, synchronize, reopened, ready_for_review]
-    paths:
-      - "*/ql/src/**/*.ql"
-      - "*/ql/src/**/*.qll"
-      - "*/ql/lib/**/*.ql"
-      - "*/ql/lib/**/*.qll"
-      - "!**/experimental/**"
-      - "!ql/**"
-      - "!swift/**"
-      - ".github/workflows/check-change-note.yml"
-
-jobs:
-  check-change-note:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Fail if no change note found. To fix, either add one, or add the `no-change-note-required` label.
-        if: |
-          github.event.pull_request.draft == false &&
-          !contains(github.event.pull_request.labels.*.name, 'no-change-note-required')
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' |
-          grep true -c
-      - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md' or 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text.
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$"))' |
-          grep true -c

.github/workflows/check-qldoc.yml

@@ -1,51 +0,0 @@
-name: "Check QLdoc coverage"
-
-on:
-  pull_request:
-    paths:
-      - "*/ql/lib/**"
-      - .github/workflows/check-qldoc.yml
-      - .github/actions/fetch-codeql/action.yml
-    branches:
-      - main
-      - "rc/*"
-
-jobs:
-  qldoc:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-
-      - name: Install CodeQL
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Check QLdoc coverage
-        shell: bash
-        run: |
-          EXIT_CODE=0
-          # TODO: remove the swift exception from the regex when we fix generated QLdoc
-          # TODO: remove the shared exception from the regex when coverage of qlpacks without dbschemes is supported
-          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!(swift|shared))[a-z]*/ql/lib' || true; } | sort -u)"
-          for pack_dir in ${changed_lib_packs}; do
-            lang="${pack_dir%/ql/lib}"
-            codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
-          done
-          git checkout HEAD^
-          for pack_dir in ${changed_lib_packs}; do
-            # When we add a new language, pack_dir would not exist in HEAD^.
-            # In this case the right thing to do is to skip the check.
-            [[ ! -d "${pack_dir}" ]] && continue
-            lang="${pack_dir%/ql/lib}"
-            codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
-            awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-current.txt" | sort -u > "${RUNNER_TEMP}/current-undocumented.txt"
-            awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-baseline.txt" | sort -u > "${RUNNER_TEMP}/baseline-undocumented.txt"
-            UNDOCUMENTED="$(grep -f <(comm -13 "${RUNNER_TEMP}/baseline-undocumented.txt" "${RUNNER_TEMP}/current-undocumented.txt") "${RUNNER_TEMP}/${lang}-current.txt" || true)"
-            if [ -n "$UNDOCUMENTED" ]; then
-              echo "$UNDOCUMENTED" | awk -F, '{gsub(/"/,""); print "::warning file='"${pack_dir}"'/"$1",line="$2"::Missing QLdoc for "$5, $3 }'
-              EXIT_CODE=1
-            fi
-          done
-          exit "${EXIT_CODE}"

.github/workflows/check-query-ids.yml

@@ -1,21 +0,0 @@
-name: Check query IDs
-
-on:
-  pull_request:
-    paths:
-      - "**/src/**/*.ql"
-      - misc/scripts/check-query-ids.py
-      - .github/workflows/check-query-ids.yml
-    branches:
-      - main
-      - "rc/*"
-  workflow_dispatch:
-
-jobs:
-  check:
-    name: Check query IDs
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check for duplicate query IDs
-        run: python3 misc/scripts/check-query-ids.py

.github/workflows/close-stale.yml

@@ -1,30 +0,0 @@
-name: Mark stale issues
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "30 1 * * *"
-
-jobs:
-  stale:
-    if: github.repository == 'github/codeql'
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/stale@v7
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Comment or remove the `Stale` label in order to avoid having this issue closed in 7 days.'
-        close-issue-message: 'This issue was closed because it has been inactive for 7 days.'
-        days-before-stale: 14
-        days-before-close: 7
-        only-labels: awaiting-response
-
-        # do not mark PRs as stale
-        days-before-pr-stale: -1
-        days-before-pr-close: -1
-
-        # Uncomment for dry-run
-        # debug-only: true
-        # operations-per-run: 1000

.github/workflows/codeql-analysis.yml

@@ -1,62 +0,0 @@
-name: "Code scanning - action"
-
-on:
-  push:
-    branches:
-      - main
-      - 'rc/*'
-  pull_request:
-    branches:
-      - main
-      - 'rc/*'
-    paths:
-      - 'csharp/**'
-      - '.github/codeql/**'
-      - '.github/workflows/codeql-analysis.yml'
-  schedule:
-    - cron: '0 9 * * 1'
-
-jobs:
-  CodeQL-Build:
-
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: read
-      security-events: write
-      pull-requests: read
-
-    steps:
-    - name: Setup dotnet
-      uses: actions/setup-dotnet@v2
-      with:
-        dotnet-version: 6.0.202
-
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@main
-      # Override language selection by uncommenting this and choosing your languages
-      with:
-        languages: csharp
-        config-file: ./.github/codeql/codeql-config.yml
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    #- name: Autobuild
-    #  uses: github/codeql-action/autobuild@main
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    - run: |
-       dotnet build csharp
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@main

.github/workflows/compile-queries.yml

@@ -1,37 +0,0 @@
-name: "Compile all queries using the latest stable CodeQL CLI"
-
-on:
-  push:
-    branches:  # makes sure the cache gets populated - running on the branches people tend to merge into.
-      - main
-      - "rc/*"
-      - "codeql-cli-*"
-  pull_request:
-
-jobs:
-  compile-queries:
-    runs-on: ubuntu-latest-xl
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup CodeQL
-        uses: ./.github/actions/fetch-codeql
-        with:
-          channel: 'release'
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: all-queries
-      - name: check formatting
-        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
-      - name: compile queries - check-only
-        # run with --check-only if running in a PR (github.sha != main)
-        if : ${{ github.event_name == 'pull_request' }}
-        shell: bash
-        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-      - name: compile queries - full
-        # do full compile if running on main - this populates the cache
-        if : ${{ github.event_name != 'pull_request' }}
-        shell: bash
-        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/csharp-qltest.yml

@@ -1,86 +0,0 @@
-name: "C#: Run QL Tests"
-
-on:
-  push:
-    paths:
-      - "csharp/**"
-      - "shared/**"
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "csharp/**"
-      - "shared/**"
-      - .github/workflows/csharp-qltest.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
-
-defaults:
-  run:
-    working-directory: csharp
-
-jobs:
-  qlupgrade:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - name: Check DB upgrade scripts
-        run: |
-          echo >empty.trap
-          codeql dataset import -S ql/lib/upgrades/initial/semmlecode.csharp.dbscheme testdb empty.trap
-          codeql dataset upgrade testdb --additional-packs ql/lib
-          diff -q testdb/semmlecode.csharp.dbscheme ql/lib/semmlecode.csharp.dbscheme
-      - name: Check DB downgrade scripts
-        run: |
-          echo >empty.trap
-          rm -rf testdb; codeql dataset import -S ql/lib/semmlecode.csharp.dbscheme testdb empty.trap
-          codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
-           --dbscheme=ql/lib/semmlecode.csharp.dbscheme --target-dbscheme=downgrades/initial/semmlecode.csharp.dbscheme |
-           xargs codeql execute upgrades testdb
-          diff -q testdb/semmlecode.csharp.dbscheme downgrades/initial/semmlecode.csharp.dbscheme
-  qltest:
-    runs-on: ubuntu-latest-xl
-    strategy:
-      fail-fast: false
-      matrix:
-        slice: ["1/2", "2/2"]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - uses: ./csharp/actions/create-extractor-pack
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with:
-          key: csharp-qltest-${{ matrix.slice }}
-      - name: Run QL tests
-        run: |
-          CODEQL_PATH=$(gh codeql version --format=json | jq -r .unpackedLocation)
-          # The legacy ASP extractor is not in this repo, so take the one from the nightly build
-          mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
-          # Safe guard against using the bundled extractor
-          rm -rf "$CODEQL_PATH/csharp"
-          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-  unit-tests:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup dotnet
-        uses: actions/setup-dotnet@v3
-        with:
-          dotnet-version: 6.0.202
-      - name: Extractor unit tests
-        run: |
-          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Util.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Extraction.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"

.github/workflows/csv-coverage-metrics.yml

@@ -1,71 +0,0 @@
-name: "Publish framework coverage as metrics"
-
-on:
-  schedule:
-    - cron: '5 0 * * *'
-  push:
-    branches:
-      - main
-  workflow_dispatch:
-  pull_request:
-    branches:
-      - main
-    paths:
-      - ".github/workflows/csv-coverage-metrics.yml"
-      - ".github/actions/fetch-codeql/action.yml"
-
-jobs:
-  publish-java:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Setup CodeQL
-        uses: ./.github/actions/fetch-codeql
-      - name: Create empty database
-        run: |
-          DATABASE="${{ runner.temp }}/java-database"
-          PROJECT="${{ runner.temp }}/java-project"
-          mkdir -p "$PROJECT/src/tmp/empty"
-          echo "class Empty {}" >> "$PROJECT/src/tmp/empty/Empty.java"
-          codeql database create "$DATABASE" --language=java --source-root="$PROJECT" --command 'javac src/tmp/empty/Empty.java'
-      - name: Capture coverage information
-        run: |
-          DATABASE="${{ runner.temp }}/java-database"
-          codeql database analyze --format=sarif-latest --output=metrics-java.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
-      - uses: actions/upload-artifact@v3
-        with:
-          name: metrics-java.sarif
-          path: metrics-java.sarif
-          retention-days: 20
-      - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@main
-        with:
-          sarif_file: metrics-java.sarif
-  
-  publish-csharp:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Setup CodeQL
-        uses: ./.github/actions/fetch-codeql
-      - name: Create empty database
-        run: |
-          DATABASE="${{ runner.temp }}/csharp-database"
-          PROJECT="${{ runner.temp }}/csharp-project"
-          dotnet new classlib --language=C# --output="$PROJECT"
-          codeql database create "$DATABASE" --language=csharp --source-root="$PROJECT" --command 'dotnet build /t:rebuild csharp-project.csproj'
-      - name: Capture coverage information
-        run: |
-          DATABASE="${{ runner.temp }}/csharp-database"
-          codeql database analyze --format=sarif-latest --output=metrics-csharp.sarif -- "$DATABASE" ./csharp/ql/src/Metrics/Summaries/FrameworkCoverage.ql
-      - uses: actions/upload-artifact@v3
-        with:
-          name: metrics-csharp.sarif
-          path: metrics-csharp.sarif
-          retention-days: 20
-      - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@main
-        with:
-          sarif_file: metrics-csharp.sarif

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -1,96 +0,0 @@
-name: Check framework coverage changes
-
-on:
-  pull_request:
-    paths:
-      - ".github/workflows/csv-coverage-pr-comment.yml"
-      - ".github/workflows/csv-coverage-pr-artifacts.yml"
-      - ".github/actions/fetch-codeql/action.yml"
-      - "*/ql/src/**/*.ql"
-      - "*/ql/src/**/*.qll"
-      - "*/ql/lib/**/*.ql"
-      - "*/ql/lib/**/*.qll"
-      - "misc/scripts/library-coverage/*.py"
-      # input data files
-      - "*/documentation/library-coverage/cwe-sink.csv"
-      - "*/documentation/library-coverage/frameworks.csv"
-    branches:
-      - main
-      - "rc/*"
-
-jobs:
-  generate:
-    name: Generate framework coverage artifacts
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Dump GitHub context
-        env:
-          GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-        run: echo "$GITHUB_CONTEXT"
-      - name: Clone self (github/codeql) - MERGE
-        uses: actions/checkout@v3
-        with:
-          path: merge
-      - name: Clone self (github/codeql) - BASE
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-          path: base
-      - run: |
-          git checkout HEAD^1
-          git log -1 --format='%H'
-        working-directory: base
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - name: Download CodeQL CLI
-        uses: ./merge/.github/actions/fetch-codeql
-      - name: Generate CSV files on merge commit of the PR
-        run: |
-          echo "Running generator on merge"
-          python merge/misc/scripts/library-coverage/generate-report.py ci merge merge
-          mkdir out_merge
-          cp framework-coverage-*.csv out_merge/
-          cp framework-coverage-*.rst out_merge/
-      - name: Generate CSV files on base commit of the PR
-        run: |
-          echo "Running generator on base"
-          python base/misc/scripts/library-coverage/generate-report.py ci base base
-          mkdir out_base
-          cp framework-coverage-*.csv out_base/
-          cp framework-coverage-*.rst out_base/
-      - name: Generate diff of coverage reports
-        run: |
-          python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
-      - name: Upload CSV package list
-        uses: actions/upload-artifact@v3
-        with:
-          name: csv-framework-coverage-merge
-          path: |
-            out_merge/framework-coverage-*.csv
-            out_merge/framework-coverage-*.rst
-      - name: Upload CSV package list
-        uses: actions/upload-artifact@v3
-        with:
-          name: csv-framework-coverage-base
-          path: |
-            out_base/framework-coverage-*.csv
-            out_base/framework-coverage-*.rst
-      - name: Upload comparison results
-        uses: actions/upload-artifact@v3
-        with:
-          name: comparison
-          path: |
-            comparison.md
-      - name: Save PR number
-        run: |
-          mkdir -p pr
-          echo ${{ github.event.pull_request.number }} > pr/NR
-      - name: Upload PR number
-        uses: actions/upload-artifact@v3
-        with:
-          name: pr
-          path: pr/

.github/workflows/csv-coverage-pr-comment.yml

@@ -1,34 +0,0 @@
-name: Comment on PR with framework coverage changes
-
-on:
-  workflow_run:
-    workflows: ["Check framework coverage changes"]
-    types:
-      - completed
-
-jobs:
-  check:
-    name: Check framework coverage differences and comment
-    runs-on: ubuntu-latest
-    if: >
-      ${{ github.event.workflow_run.event == 'pull_request' &&
-      github.event.workflow_run.conclusion == 'success' }}
-
-    steps:
-    - name: Dump GitHub context
-      env:
-        GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-      run: echo "$GITHUB_CONTEXT"
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.8
-
-    - name: Check coverage difference file and comment
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        RUN_ID: ${{ github.event.workflow_run.id }}
-      run: |
-        python misc/scripts/library-coverage/comment-pr.py "$GITHUB_REPOSITORY" "$RUN_ID"

.github/workflows/csv-coverage-timeseries.yml

@@ -1,33 +0,0 @@
-name: Build framework coverage timeseries reports
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
-        with:
-          path: script
-      - name: Clone self (github/codeql) for analysis
-        uses: actions/checkout@v3
-        with:
-          path: codeqlModels
-          fetch-depth: 0
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - name: Download CodeQL CLI
-        uses: ./script/.github/actions/fetch-codeql
-      - name: Build modeled package list
-        run: |
-          python script/misc/scripts/library-coverage/generate-timeseries.py codeqlModels
-      - name: Upload timeseries CSV
-        uses: actions/upload-artifact@v3
-        with:
-          name: framework-coverage-timeseries
-          path: framework-coverage-timeseries-*.csv

.github/workflows/csv-coverage-update.yml

@@ -1,38 +0,0 @@
-name: Update framework coverage reports
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "0 0 * * *"
-
-jobs:
-  update:
-    name: Update framework coverage report
-    if: github.repository == 'github/codeql'
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Dump GitHub context
-        env:
-          GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-        run: echo "$GITHUB_CONTEXT"
-      - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
-        with:
-          path: ql
-          fetch-depth: 0
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - name: Download CodeQL CLI
-        uses: ./ql/.github/actions/fetch-codeql
-      - name: Generate coverage files
-        run: |
-          python ql/misc/scripts/library-coverage/generate-report.py ci ql ql
-
-      - name: Create pull request with changes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          python ql/misc/scripts/library-coverage/create-pr.py ql "$GITHUB_REPOSITORY"

.github/workflows/csv-coverage.yml

@@ -1,42 +0,0 @@
-name: Build framework coverage reports
-
-on:
-  workflow_dispatch:
-    inputs:
-      qlModelShaOverride:
-        description: "github/codeql repo SHA used for looking up the CSV models"
-        required: false
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
-        with:
-          path: script
-      - name: Clone self (github/codeql) for analysis
-        uses: actions/checkout@v3
-        with:
-          path: codeqlModels
-          ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - name: Download CodeQL CLI
-        uses: ./script/.github/actions/fetch-codeql
-      - name: Build modeled package list
-        run: |
-          python script/misc/scripts/library-coverage/generate-report.py ci codeqlModels script
-      - name: Upload CSV package list
-        uses: actions/upload-artifact@v3
-        with:
-          name: framework-coverage-csv
-          path: framework-coverage-*.csv
-      - name: Upload RST package list
-        uses: actions/upload-artifact@v3
-        with:
-          name: framework-coverage-rst
-          path: framework-coverage-*.rst

.github/workflows/dataset_measure.yml

@@ -0,0 +1,71 @@
+name: Collect database stats
+
+on:
+  push:
+    branches:
+      - main
+      - 'rc/*'
+    paths:
+      - ql/lib/ruby.dbscheme
+  pull_request:
+    branches:
+      - main
+      - 'rc/*'
+    paths:
+      - ql/lib/ruby.dbscheme
+  workflow_dispatch:
+
+jobs:
+  measure:
+    env:
+      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
+    strategy:
+      fail-fast: false
+      matrix:
+        repo: [rails/rails, discourse/discourse, spree/spree]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - uses: ./.github/actions/create-extractor-pack
+
+      - name: Checkout ${{ matrix.repo }}
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ matrix.repo }}
+          path: ${{ github.workspace }}/repo
+      - name: Create database
+        run: |
+          codeql/codeql database create \
+            --search-path "${{ github.workspace }}" \
+            --threads 4 \
+            --language ruby --source-root "${{ github.workspace }}/repo" \
+            "${{ runner.temp }}/database"
+      - name: Measure database
+        run: |
+          mkdir -p "stats/${{ matrix.repo }}"
+          codeql/codeql dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ruby"
+      - uses: actions/upload-artifact@v2
+        with:
+          name: measurements
+          path: stats
+          retention-days: 1
+
+  merge:
+    runs-on: ubuntu-latest
+    needs: measure
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+        with:
+          name: measurements
+          path: stats
+      - run: |
+          python -m pip install --user lxml
+          find stats -name 'stats.xml' | sort | xargs python scripts/merge_stats.py --output ql/lib/ruby.dbscheme.stats --normalise ruby_tokeninfo
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ruby.dbscheme.stats
+          path: ql/lib/ruby.dbscheme.stats

.github/workflows/go-tests-other-os.yml

@@ -1,80 +0,0 @@
-name: "Go: Run Tests - Other OS"
-on:
-  pull_request:
-    paths:
-      - "go/**"
-      - "!go/ql/**" # don't run other-os if only ql/ files changed
-      - .github/workflows/go-tests-other-os.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-jobs:
-  test-mac:
-    name: Test MacOS
-    runs-on: macos-latest
-    steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.19
-        id: go
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up CodeQL CLI
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Enable problem matchers in repository
-        shell: bash
-        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-      - name: Build
-        run: |
-          cd go
-          make
-
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with:
-          key: go-qltest
-      - name: Test
-        run: |
-          cd go
-          make test cache="${{ steps.query-cache.outputs.cache-dir }}"
-
-  test-win:
-    name: Test Windows
-    runs-on: windows-latest-xl
-    steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.19
-        id: go
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up CodeQL CLI
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Enable problem matchers in repository
-        shell: bash
-        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-      - name: Build
-        run: |
-          cd go
-          make
-
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with:
-          key: go-qltest
-
-      - name: Test
-        run: |
-          cd go
-          make test cache="${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/go-tests.yml

@@ -1,69 +0,0 @@
-name: "Go: Run Tests"
-on:
-  push:
-    paths:
-      - "go/**"
-      - .github/workflows/go-tests.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "go/**"
-      - .github/workflows/go-tests.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-jobs:
-  test-linux:
-    name: Test Linux (Ubuntu)
-    runs-on: ubuntu-latest-xl
-    steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.19
-        id: go
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up CodeQL CLI
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Enable problem matchers in repository
-        shell: bash
-        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-      - name: Build
-        run: |
-          cd go
-          make
-
-      - name: Check that all Go code is autoformatted
-        run: |
-          cd go
-          make check-formatting
-
-      - name: Compile qhelp files to markdown
-        run: |
-          cd go
-          env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown
-
-      - name: Upload qhelp markdown
-        uses: actions/upload-artifact@v3
-        with:
-          name: qhelp-markdown
-          path: go/qhelp-out/**/*.md
-
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with:
-          key: go-qltest
-
-      - name: Test
-        run: |
-          cd go
-          make test cache="${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/js-ml-tests.yml

@@ -1,65 +0,0 @@
-name: JS ML-powered queries tests
-
-on:
-  push:
-    paths:
-      - "javascript/ql/experimental/adaptivethreatmodeling/**"
-      - .github/workflows/js-ml-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "javascript/ql/experimental/adaptivethreatmodeling/**"
-      - .github/workflows/js-ml-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-  workflow_dispatch:
-
-defaults:
-  run:
-    working-directory: javascript/ql/experimental/adaptivethreatmodeling
-
-jobs:
-  qltest:
-    name: Test QL
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: ./.github/actions/fetch-codeql
-
-      - name: Install pack dependencies
-        run: |
-          for pack in modelbuilding src test; do
-            codeql pack install --mode verify -- "${pack}"
-          done
-      
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: js-ml-test
-
-      - name: Check QL compilation
-        run: |
-          codeql query compile \
-            --check-only \
-            --ram 50000 \
-            --additional-packs "${{ github.workspace }}" \
-            --threads=0 \
-            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
-            -- \
-            lib modelbuilding src
-
-      - name: Run QL tests
-        run: |
-          codeql test run \
-            --threads=0 \
-            --ram 50000 \
-            --additional-packs "${{ github.workspace }}" \
-            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
-            -- \
-            test

.github/workflows/labeler.yml

@@ -1,14 +0,0 @@
-name: "Pull Request Labeler"
-on:
-- pull_request_target
-
-jobs:
-  triage:
-    permissions:
-      contents: read
-      pull-requests: write
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/labeler@v4
-      with:
-        repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/mad_modelDiff.yml

@@ -1,105 +0,0 @@
-name: Models as Data - Diff
-
-on:
-  workflow_dispatch:
-    inputs:
-      projects:
-        description: "The projects to generate models for"
-        required: true
-        default: '["netty/netty"]'
-  pull_request:
-    branches:
-      - main
-    paths:
-      - "java/ql/src/utils/modelgenerator/**/*.*"
-      - ".github/workflows/mad_modelDiff.yml"
-
-permissions:
-  contents: read
-
-jobs:
-  model-diff:
-    name: Model Difference
-    runs-on: ubuntu-latest
-    if: github.repository == 'github/codeql'
-    strategy:
-      matrix:
-        slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
-    steps:
-      - name: Clone github/codeql from PR
-        uses: actions/checkout@v3
-        if: github.event.pull_request
-        with:
-          path: codeql-pr
-      - name: Clone github/codeql from main
-        uses: actions/checkout@v3
-        with:
-          path: codeql-main
-          ref: main
-      - uses: ./codeql-main/.github/actions/fetch-codeql
-      - name: Download database
-        env:
-          SLUG: ${{ matrix.slug }}
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          set -x
-          mkdir lib-dbs
-          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
-          gh api -H "Accept: application/zip" "/repos/${SLUG}/code-scanning/codeql/databases/java" > "$SHORTNAME.zip"
-          unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
-          mkdir "lib-dbs/$SHORTNAME/"
-          mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/"
-      - name: Generate Models (PR and main)
-        run: |
-          set -x
-          mkdir tmp-models
-          MODELS=`pwd`/tmp-models
-          DATABASES=`pwd`/lib-dbs
-
-          analyzeDatabaseWithCheckout() {
-            QL_VARIANT=$1
-            DATABASE=$2
-            cd codeql-$QL_VARIANT
-            SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE ${SHORTNAME}.temp.model.yml
-            mv java/ql/lib/ext/generated/${SHORTNAME}.temp.model.yml $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.model.yml
-            cd ..
-          }
-
-          for d in $DATABASES/*/ ; do
-            ls -1 "$d"
-
-            analyzeDatabaseWithCheckout "main" $d
-            if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]
-            then
-              analyzeDatabaseWithCheckout "pr" $d
-            fi
-          done
-      - name: Install diff2html
-        if: github.event.pull_request
-        run: |
-          npm install -g diff2html-cli
-      - name: Generate Model Diff
-        if: github.event.pull_request
-        run: |
-          set -x
-          MODELS=`pwd`/tmp-models
-          ls -1 tmp-models/
-          for m in $MODELS/*_main.model.yml ; do
-            t="${m/main/"pr"}"
-            basename=`basename $m`
-            name="diff_${basename/_main.model.yml/""}"
-            (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
-          done
-      - uses: actions/upload-artifact@v3
-        with:
-          name: models
-          path: tmp-models/*.model.yml
-          retention-days: 20
-      - uses: actions/upload-artifact@v3
-        with:
-          name: diffs
-          path: tmp-models/*.html
-          # An html file is only produced if the generated models differ.
-          if-no-files-found: ignore
-          retention-days: 20

.github/workflows/mad_regenerate-models.yml

@@ -1,63 +0,0 @@
-name: Regenerate framework models
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "30 2 * * *"
-  pull_request:
-    branches:
-      - main
-    paths:
-      - ".github/workflows/mad_regenerate-models.yml"
-      - ".github/actions/fetch-codeql/action.yml"
-
-jobs:
-  regenerate-models:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        # placeholder required for each axis, excluded below, replaced by the actual combinations (see include)
-        slug: ["placeholder"]
-        ref: ["placeholder"]
-        include:
-          - slug: "apache/commons-io"
-            ref: "13258ce2d07aa0e764bbaa8020af4dcd3a02a620"
-        exclude:
-          - slug: "placeholder"
-            ref: "placeholder"
-    steps:
-      - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
-      - name: Setup CodeQL binaries
-        uses: ./.github/actions/fetch-codeql
-      - name: Clone repositories
-        uses: actions/checkout@v3
-        with:
-          path: repos/${{ matrix.ref }}
-          ref: ${{ matrix.ref }}
-          repository: ${{ matrix.slug }}
-      - name: Build database
-        env:
-          SLUG: ${{ matrix.slug }}
-          REF: ${{ matrix.ref }}
-        run: |
-          mkdir dbs
-          cd repos/${REF}
-          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
-          codeql database create --language=java ../../dbs/${SHORTNAME}
-      - name: Regenerate models in-place
-        env:
-          SLUG: ${{ matrix.slug }}
-        run: |
-          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
-          java/ql/src/utils/modelgenerator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
-      - name: Stage changes
-        run: |
-          find java -name "*.model.yml" -print0 | xargs -0 git add
-          git status
-          git diff --cached > models.patch
-      - uses: actions/upload-artifact@v3
-        with:
-          name: patch
-          path: models.patch
-          retention-days: 7

.github/workflows/post-pr-comment.yml

@@ -1,74 +0,0 @@
-# This workflow is the second part of the process described in
-# .github/workflows/qhelp-pr-preview.yml
-# See that file for more info.
-
-name: Post PR comment
-on:
-  workflow_run:
-    workflows: [Render QHelp changes]
-    types:
-      - completed
-
-permissions:
-  pull-requests: write
-  actions: read
-
-jobs:
-  post_comment:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download artifact
-        run: gh run download "${WORKFLOW_RUN_ID}" --repo "${GITHUB_REPOSITORY}" --name "comment"
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-          WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
-
-      - name: Check that PR SHA matches workflow SHA
-        run: |
-          PR="$(grep -o '^[0-9]\+$' pr_number.txt)"
-          PR_HEAD_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/pulls/${PR}" --jq .head.sha)"
-          # Check that the pull-request head SHA matches the head SHA of the workflow run
-          if [ "${WORKFLOW_RUN_HEAD_SHA}" != "${PR_HEAD_SHA}" ]; then
-            echo "PR head SHA ${PR_HEAD_SHA} does not match workflow_run event SHA ${WORKFLOW_RUN_HEAD_SHA}. Stopping." 1>&2
-            exit 1
-          fi
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-          WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_commit.id }}
-
-      - name: Create or update comment
-        run: |
-          COMMENT_PREFIX="QHelp previews"
-          COMMENT_AUTHOR="github-actions[bot]"
-          PR_NUMBER="$(grep -o '^[0-9]\+$' pr_number.txt)"
-
-          # If there is no existing comment, comment_id.txt will contain just a
-          # newline (due to jq & gh behaviour). This will cause grep to fail, so
-          # we catch that.
-          RAW_COMMENT_ID=$(grep -o '^[0-9]\+$' comment_id.txt || true)
-
-          if [ $RAW_COMMENT_ID ]
-          then
-            # Fetch existing comment, and validate:
-            # - comment belongs to the PR with number $PR_NUMBER
-            # - comment starts with the expected prefix ("QHelp previews")
-            # - comment author is github-actions[bot]
-            FILTER='select(.issue_url | endswith($repo+"/issues/"+$pr))
-                  | select(.body | startswith($prefix))
-                  | select(.user.login == $author)
-                  | .id'
-            COMMENT_ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${RAW_COMMENT_ID}" | jq --arg repo "${GITHUB_REPOSITORY}" --arg pr "${PR_NUMBER}" --arg prefix "${COMMENT_PREFIX}" --arg author "${COMMENT_AUTHOR}" "${FILTER}")
-            if [ $COMMENT_ID ]
-            then
-              # Update existing comment
-              jq --rawfile body comment_body.txt '{"body":$body}' -n | gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${COMMENT_ID}" -X PATCH --input -
-            else
-              echo "Comment ${RAW_COMMENT_ID} did not pass validations: not editing." >&2
-              exit 1
-            fi
-          else
-            # Create new comment
-            jq --rawfile body comment_body.txt '{"body":$body}' -n | gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -X POST --input -
-          fi
-        env:
-          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/qhelp-pr-preview.yml

@@ -1,102 +0,0 @@
-# This workflow checks for any changes in .qhelp files in pull requests.
-# For any changed files, it renders them to markdown in a file called `comment_body.txt`.
-# It then checks if there's an existing comment on the pull request generated by
-# this workflow, and writes the comment ID to `comment_id.txt`.
-# It also writes the PR number to `pr_number.txt`.
-# These three files are uploaded as an artifact.
-
-# When this workflow completes, the workflow "Post PR comment" runs.
-# It downloads the artifact and adds a comment to the PR with the rendered
-# QHelp.
-
-# The task is split like this because creating PR comments requires extra
-# permissions that we don't want to expose to PRs from external forks.
-
-# For more info see:
-# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_run
-# https://docs.github.com/en/actions/security-guides/automatic-token-authentication#permissions-for-the-github_token
-name: Render QHelp changes
-
-permissions:
-  contents: read
-  pull-requests: read
-
-on:
-  pull_request:
-    branches:
-      - main
-      - "rc/*"
-    paths:
-      - "**/*.qhelp"
-
-jobs:
-  qhelp:
-    runs-on: ubuntu-latest
-    steps:
-      - run: echo "${PR_NUMBER}" > pr_number.txt
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-      - uses: actions/upload-artifact@v3
-        with:
-          name: comment
-          path: pr_number.txt
-          if-no-files-found: error
-          retention-days: 1
-
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-          persist-credentials: false
-      - uses: ./.github/actions/fetch-codeql
-      - name: Determine changed files
-        id: changes
-        run: |
-          (git diff -z --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep -z '.qhelp$' | grep -z -v '.inc.qhelp';
-           git diff -z --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep -z '.inc.qhelp$' | xargs --null -rn1 basename -z | xargs --null -rn1 git grep -z -l) |
-           grep -z '.qhelp$' | grep -z -v '^-' | sort -z -u > "${RUNNER_TEMP}/paths.txt"
-
-      - name: QHelp preview
-        run: |
-          EXIT_CODE=0
-          echo "QHelp previews:" > comment_body.txt
-          while read -r -d $'\0' path; do
-            if [ ! -f "${path}" ]; then
-               exit 1
-            fi
-            echo "<details> <summary>${path}</summary>"
-            echo
-            codeql generate query-help --format=markdown -- "./${path}" 2> errors.txt || EXIT_CODE="$?"
-            if [ -s errors.txt ]; then
-               echo "# errors/warnings:"
-               echo '```'
-               cat errors.txt
-               cat errors.txt 1>&2
-               echo '```'
-            fi
-            echo "</details>"
-          done < "${RUNNER_TEMP}/paths.txt" >> comment_body.txt
-          exit "${EXIT_CODE}"
-
-      - if: always()
-        uses: actions/upload-artifact@v3
-        with:
-          name: comment
-          path: comment_body.txt
-          if-no-files-found: error
-          retention-days: 1
-
-      - name: Save ID of existing QHelp comment (if it exists)
-        run: |
-          # Find the latest comment starting with "QHelp previews"
-          COMMENT_PREFIX="QHelp previews"
-          gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate | jq --arg prefix "${COMMENT_PREFIX}" '[.[] | select(.body|startswith($prefix)) | .id] | max' > comment_id.txt
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-          PR_NUMBER: ${{ github.event.number }}
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: comment
-          path: comment_id.txt
-          if-no-files-found: error
-          retention-days: 1

.github/workflows/qhelp.yml

@@ -0,0 +1,39 @@
+name: Query help preview
+
+on:
+  pull_request:
+    branches:
+      - main
+      - 'rc/*'
+    paths:
+      - "**/*.qhelp"
+
+jobs:
+  qhelp:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Determine changed files
+        id: changes
+        run: |
+          echo -n "::set-output name=qhelp_files::"
+          (git diff --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep .qhelp$ | grep -v .inc.qhelp;
+           git diff --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep .inc.qhelp$ | xargs -d '\n' -rn1 basename | xargs -d '\n' -rn1 git grep -l) |
+           sort -u | xargs -d '\n' -n1 printf "'%s' "
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: QHelp preview
+        if: ${{ steps.changes.outputs.qhelp_files }}
+        run: |
+          ( echo "QHelp previews:";
+          for path in ${{ steps.changes.outputs.qhelp_files }} ; do
+            echo "<details> <summary>${path}</summary>"
+            echo
+            codeql/codeql generate query-help --format=markdown ${path}
+            echo "</details>"
+          done) | gh pr comment "${{ github.event.pull_request.number }}" -F -
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/ql-for-ql-build.yml

@@ -1,177 +0,0 @@
-name: Run QL for QL
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-    paths:
-      - "ql/**"
-      - "**.qll"
-      - "**.ql"
-      - "**.dbscheme"
-      - "**/qlpack.yml"
-      - ".github/workflows/ql-for-ql-build.yml"
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  analyze:
-    runs-on: ubuntu-latest-xl
-    steps:
-      ### Build the queries ###
-      - uses: actions/checkout@v3
-      - name: Find latest bundle
-        id: find-latest-bundle
-        uses: ./.github/actions/find-latest-bundle
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
-        with:
-          languages: javascript # does not matter
-          tools: ${{ steps.find-latest-bundle.outputs.url }}
-      - name: Get CodeQL version
-        id: get-codeql-version
-        run: |
-          echo "version=$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)" >> $GITHUB_OUTPUT
-        shell: bash
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - uses: ./.github/actions/os-version
-        id: os_version
-      - name: Cache entire pack
-        id: cache-pack
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/pack
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-pack-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
-      - name: Cache queries
-        if: steps.cache-pack.outputs.cache-hit != 'true'
-        id: cache-queries
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/queries
-          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
-      - name: Build query pack
-        if: steps.cache-queries.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: |
-          cd ql/ql/src
-          "${CODEQL}" pack create -j 16
-          mv .codeql/pack/codeql/ql/0.0.0 ${{ runner.temp }}/queries
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Move cache queries to pack
-        if: steps.cache-pack.outputs.cache-hit != 'true'
-        run: |
-          cp -r ${{ runner.temp }}/queries ${{ runner.temp }}/pack
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-
-      ### Build the extractor ###
-      - name: Cache entire extractor
-        if: steps.cache-pack.outputs.cache-hit != 'true'
-        id: cache-extractor
-        uses: actions/cache@v3
-        with:
-          path: |
-            ql/target/release/ql-autobuilder
-            ql/target/release/ql-autobuilder.exe
-            ql/target/release/ql-extractor
-            ql/target/release/ql-extractor.exe
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
-      - name: Cache cargo
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
-      - name: Check formatting
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: cd ql; cargo fmt --all -- --check
-      - name: Build
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: cd ql; cargo build --verbose
-      - name: Run tests
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: cd ql; cargo test --verbose
-      - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: cd ql; cargo build --release
-      - name: Generate dbscheme
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
-
-      ### Package the queries and extractor ###
-      - name: Package pack
-        if: steps.cache-pack.outputs.cache-hit != 'true'
-        run: |
-          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats ${PACK}/
-          mkdir -p ${PACK}/tools/linux64
-          cp ql/target/release/ql-autobuilder  ${PACK}/tools/linux64/autobuilder
-          cp ql/target/release/ql-extractor ${PACK}/tools/linux64/extractor
-          chmod +x ${PACK}/tools/linux64/autobuilder
-          chmod +x ${PACK}/tools/linux64/extractor
-        env:
-          PACK: ${{ runner.temp }}/pack
-
-      ### Run the analysis ###
-      - name: Hack codeql-action options
-        run: |
-          JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .resolve.languages=["--search-path", $pack] | .database.init=["--search-path", $pack]')
-          echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
-        env:
-          PACK: ${{ runner.temp }}/pack
-
-      - name: Create CodeQL config file
-        run: |
-          echo "paths-ignore:" >> ${CONF}
-          echo "  - ql/ql/test" >> ${CONF}
-          echo "  - \"*/ql/lib/upgrades/\"" >> ${CONF}
-          echo "disable-default-queries: true" >> ${CONF}
-          echo "queries:" >> ${CONF}
-          echo "  - uses: ./ql/ql/src/codeql-suites/ql-code-scanning.qls" >> ${CONF}
-          echo "Config file: "
-          cat ${CONF}
-        env:
-          CONF: ./ql-for-ql-config.yml
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
-        with:
-          languages: ql
-          db-location: ${{ runner.temp }}/db
-          config-file: ./ql-for-ql-config.yml
-          tools: ${{ steps.find-latest-bundle.outputs.url }}
-      - name: Move pack queries
-        run: |
-          cp -r ${PACK}/queries ql/ql/src
-        env:
-          PACK: ${{ runner.temp }}/pack
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@45955cb1830b640e2c1603ad72ad542a49d47b96
-        with:
-          category: "ql-for-ql"
-      - name: Copy sarif file to CWD
-        run: cp ../results/ql.sarif ./ql-for-ql.sarif
-      - name: Fixup the $scema in sarif  # Until https://github.com/microsoft/sarif-vscode-extension/pull/436/ is part in a stable release
-        run: |
-          sed -i 's/\$schema.*/\$schema": "https:\/\/raw.githubusercontent.com\/oasis-tcs\/sarif-spec\/master\/Schemata\/sarif-schema-2.1.0",/' ql-for-ql.sarif
-      - name: Sarif as artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: ql-for-ql.sarif
-          path: ql-for-ql.sarif
-      - name: Split out the sarif file into langs
-        run: |
-          mkdir split-sarif
-          node ./ql/scripts/split-sarif.js ql-for-ql.sarif split-sarif
-      - name: Upload langs as artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: ql-for-ql-langs
-          path: split-sarif
-          retention-days: 1

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -1,85 +0,0 @@
-name: Collect database stats for QL for QL
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - ql/ql/src/ql.dbscheme
-  pull_request:
-    branches: [main]
-    paths:
-      - ql/ql/src/ql.dbscheme
-  workflow_dispatch:
-
-jobs:
-  measure:
-    env:
-      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
-    strategy:
-      matrix:
-        repo:
-          - github/codeql
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
-        with:
-          languages: javascript # does not matter
-      - uses: ./.github/actions/os-version
-        id: os_version
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
-      - name: Build Extractor
-        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./scripts/create-extractor-pack.sh
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v3
-        with:
-          repository: ${{ matrix.repo }}
-          path: ${{ github.workspace }}/repo
-      - name: Create database
-        run: |
-          "${CODEQL}" database create \
-            --search-path "ql/extractor-pack" \
-            --threads 4 \
-            --language ql --source-root "${{ github.workspace }}/repo" \
-            "${{ runner.temp }}/database"
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Measure database
-        run: |
-          mkdir -p "stats/${{ matrix.repo }}"
-          "${CODEQL}" dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ql"
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - uses: actions/upload-artifact@v3
-        with:
-          name: measurements
-          path: stats
-          retention-days: 1
-
-  merge:
-    runs-on: ubuntu-latest
-    needs: measure
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
-        with:
-          name: measurements
-          path: stats
-      - run: |
-          python -m pip install --user lxml
-          find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ruby/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ql.dbscheme.stats
-          path: ql/ql/src/ql.dbscheme.stats

.github/workflows/ql-for-ql-tests.yml

@@ -1,105 +0,0 @@
-name: Run QL for QL Tests
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - "ql/**"
-      - codeql-workspace.yml
-  pull_request:
-    branches: [main]
-    paths:
-      - "ql/**"
-      - codeql-workspace.yml
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  qltest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@45955cb1830b640e2c1603ad72ad542a49d47b96
-        with:
-          languages: javascript # does not matter
-      - uses: ./.github/actions/os-version
-        id: os_version
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/rust-toolchain.toml', 'ql/**/Cargo.lock') }}
-      - name: Build extractor
-        run: |
-          cd ql;
-          codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
-          env "PATH=$PATH:$codeqlpath" ./scripts/create-extractor-pack.sh
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: ql-for-ql-tests
-      - name: Run QL tests
-        run: |
-          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-
-  other-os: 
-    strategy:
-      matrix:
-        os: [macos-latest, windows-latest]
-    needs: [qltest]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install GNU tar 
-        if: runner.os == 'macOS'
-        run: |
-          brew install gnu-tar
-          echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
-        with:
-          languages: javascript # does not matter
-      - uses: ./.github/actions/os-version
-        id: os_version
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/rust-toolchain.toml', 'ql/**/Cargo.lock') }}
-      - name: Build extractor
-        if: runner.os != 'Windows'
-        run: |
-          cd ql;
-          codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
-          env "PATH=$PATH:$codeqlpath" ./scripts/create-extractor-pack.sh
-      - name: Build extractor (Windows)
-        if: runner.os == 'Windows'
-        shell: pwsh
-        run: |
-          cd ql;
-          $Env:PATH += ";$(dirname ${{ steps.find-codeql.outputs.codeql-path }})"
-          pwsh ./scripts/create-extractor-pack.ps1
-      - name: Run a single QL tests - Unix
-        if: runner.os != 'Windows'
-        run: |
-          "${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Run a single QL tests - Windows
-        if: runner.os == 'Windows'
-        shell: pwsh
-        run: |
-          $Env:PATH += ";$(dirname ${{ steps.find-codeql.outputs.codeql-path }})"
-          codeql test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
-      

.github/workflows/qltest.yml

@@ -0,0 +1,42 @@
+name: Run QL Tests
+
+on:
+  push:
+    branches:
+      - main
+      - 'rc/*'
+  pull_request:
+    branches:
+      - main
+      - 'rc/*'
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  qltest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/fetch-codeql
+      - uses: ./.github/actions/create-extractor-pack
+      - name: Run QL tests
+        run: |
+          codeql/codeql pack install ql/test
+          codeql/codeql test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --additional-packs "${HOME}/.codeql/packages/codeql/suite-helpers/0.0.1" --consistency-queries ql/consistency-queries ql/test
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - name: Check QL formatting
+        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql/codeql query format --check-only
+      - name: Check QL compilation
+        run: |
+          codeql/codeql pack install ql/src
+          codeql/codeql query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}" "ql/src" "ql/examples"
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - name: Check DB upgrade scripts
+        run: |
+          echo >empty.trap
+          codeql/codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
+          codeql/codeql dataset upgrade testdb --additional-packs ql/lib/upgrades
+          diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme

.github/workflows/query-list.yml

@@ -1,40 +0,0 @@
-name: Build code scanning query list
-
-on:
-  push:
-    branches:
-     - main
-     - 'rc/**'
-    tags:
-     - 'codeql-cli/*'
-  pull_request:
-    paths:
-      - '.github/workflows/query-list.yml'
-      - '.github/actions/fetch-codeql/action.yml'
-      - 'misc/scripts/generate-code-scanning-query-list.py'
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
-      with:
-        path: codeql 
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      # Look under the `codeql` directory, as this is where we checked out the `github/codeql` repo
-      uses: ./codeql/.github/actions/fetch-codeql
-    - name: Build code scanning query list
-      run: |
-        python codeql/misc/scripts/generate-code-scanning-query-list.py > code-scanning-query-list.csv
-    - name: Upload code scanning query list
-      uses: actions/upload-artifact@v3
-      with:
-        name: code-scanning-query-list
-        path: code-scanning-query-list.csv

.github/workflows/ruby-build.yml

@@ -1,228 +0,0 @@
-name: "Ruby: Build"
-
-on:
-  push:
-    paths:
-      - "ruby/**"
-      - .github/workflows/ruby-build.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "ruby/**"
-      - .github/workflows/ruby-build.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
-  workflow_dispatch:
-    inputs:
-      tag:
-        description: "Version tag to create"
-        required: false
-
-env:
-  CARGO_TERM_COLOR: always
-
-defaults:
-  run:
-    working-directory: ruby
-
-jobs:
-  build:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install GNU tar
-        if: runner.os == 'macOS'
-        run: |
-          brew install gnu-tar
-          echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
-      - uses: ./.github/actions/os-version
-        id: os_version
-      - name: Cache entire extractor
-        uses: actions/cache@v3
-        id: cache-extractor
-        with:
-          path: |
-            ruby/target/release/ruby-autobuilder
-            ruby/target/release/ruby-autobuilder.exe
-            ruby/target/release/ruby-extractor
-            ruby/target/release/ruby-extractor.exe
-            ruby/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}--${{ hashFiles('ruby/**/*.rs') }}
-      - uses: actions/cache@v3
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ruby/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-rust-cargo-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}
-      - name: Check formatting
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cargo fmt --all -- --check
-      - name: Build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cargo build --verbose
-      - name: Run tests
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cargo test --verbose
-      - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cargo build --release
-      - name: Generate dbscheme
-        if: ${{ matrix.os == 'ubuntu-latest' && steps.cache-extractor.outputs.cache-hit != 'true'}}
-        run: target/release/ruby-generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v3
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        with:
-          name: ruby.dbscheme
-          path: ruby/ql/lib/ruby.dbscheme
-      - uses: actions/upload-artifact@v3
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        with:
-          name: TreeSitter.qll
-          path: ruby/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v3
-        with:
-          name: extractor-${{ matrix.os }}
-          path: |
-            ruby/target/release/ruby-autobuilder
-            ruby/target/release/ruby-autobuilder.exe
-            ruby/target/release/ruby-extractor
-            ruby/target/release/ruby-extractor.exe
-          retention-days: 1
-  compile-queries:
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v3
-      - name: Fetch CodeQL
-        uses: ./.github/actions/fetch-codeql
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: ruby-build
-      - name: Build Query Pack
-        run: |
-          rm -rf target/packs
-          codeql pack create ../misc/suite-helpers --output target/packs
-          codeql pack create ../shared/regex --output target/packs
-          codeql pack create ../shared/ssa --output target/packs
-          codeql pack create ../shared/tutorial --output target/packs
-          codeql pack create ql/lib --output target/packs
-          codeql pack create -j0 ql/src --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-          PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
-          codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
-          (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
-      - uses: actions/upload-artifact@v3
-        with:
-          name: codeql-ruby-queries
-          path: |
-            ruby/target/packs/*
-          retention-days: 1
-
-  package:
-    runs-on: ubuntu-latest
-    needs: [build, compile-queries]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
-        with:
-          name: ruby.dbscheme
-          path: ruby/ruby
-      - uses: actions/download-artifact@v3
-        with:
-          name: extractor-ubuntu-latest
-          path: ruby/linux64
-      - uses: actions/download-artifact@v3
-        with:
-          name: extractor-windows-latest
-          path: ruby/win64
-      - uses: actions/download-artifact@v3
-        with:
-          name: extractor-macos-latest
-          path: ruby/osx64
-      - run: |
-          mkdir -p ruby
-          cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
-          mkdir -p ruby/tools/{linux64,osx64,win64}
-          cp linux64/ruby-autobuilder ruby/tools/linux64/autobuilder
-          cp osx64/ruby-autobuilder ruby/tools/osx64/autobuilder
-          cp win64/ruby-autobuilder.exe ruby/tools/win64/autobuilder.exe
-          cp linux64/ruby-extractor ruby/tools/linux64/extractor
-          cp osx64/ruby-extractor ruby/tools/osx64/extractor
-          cp win64/ruby-extractor.exe ruby/tools/win64/extractor.exe
-          chmod +x ruby/tools/{linux64,osx64}/{autobuilder,extractor}
-          zip -rq codeql-ruby.zip ruby
-      - uses: actions/upload-artifact@v3
-        with:
-          name: codeql-ruby-pack
-          path: ruby/codeql-ruby.zip
-          retention-days: 1
-      - uses: actions/download-artifact@v3
-        with:
-          name: codeql-ruby-queries
-          path: ruby/qlpacks
-      - run: |
-          echo '{
-            "provide": [
-            "ruby/codeql-extractor.yml",
-            "qlpacks/*/*/*/qlpack.yml"
-            ]
-          }' > .codeqlmanifest.json
-          zip -rq codeql-ruby-bundle.zip .codeqlmanifest.json ruby qlpacks
-      - uses: actions/upload-artifact@v3
-        with:
-          name: codeql-ruby-bundle
-          path: ruby/codeql-ruby-bundle.zip
-          retention-days: 1
-
-  test:
-    defaults:
-      run:
-        working-directory: ${{ github.workspace }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-
-    runs-on: ${{ matrix.os }}
-    needs: [package]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Fetch CodeQL
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Download Ruby bundle
-        uses: actions/download-artifact@v3
-        with:
-          name: codeql-ruby-bundle
-          path: ${{ runner.temp }}
-      - name: Unzip Ruby bundle
-        shell: bash
-        run: unzip -q -d "${{ runner.temp }}/ruby-bundle" "${{ runner.temp }}/codeql-ruby-bundle.zip"
-
-      - name: Run QL test
-        shell: bash
-        run: |
-          codeql test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" ruby/ql/test/library-tests/ast/constants/
-      - name: Create database
-        shell: bash
-        run: |
-          codeql database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root ruby/ql/test/library-tests/ast/constants/ ../database
-      - name: Analyze database
-        shell: bash
-        run: |
-          codeql database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/ruby-dataset-measure.yml

@@ -1,73 +0,0 @@
-name: "Ruby: Collect database stats"
-
-on:
-  push:
-    branches:
-      - main
-      - "rc/*"
-    paths:
-      - ruby/ql/lib/ruby.dbscheme
-      - .github/workflows/ruby-dataset-measure.yml
-  pull_request:
-    branches:
-      - main
-      - "rc/*"
-    paths:
-      - ruby/ql/lib/ruby.dbscheme
-      - .github/workflows/ruby-dataset-measure.yml
-  workflow_dispatch:
-
-jobs:
-  measure:
-    env:
-      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
-    strategy:
-      fail-fast: false
-      matrix:
-        repo: [rails/rails, discourse/discourse, spree/spree, ruby/ruby]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: ./.github/actions/fetch-codeql
-
-      - uses: ./ruby/actions/create-extractor-pack
-
-      - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v3
-        with:
-          repository: ${{ matrix.repo }}
-          path: ${{ github.workspace }}/repo
-      - name: Create database
-        run: |
-          codeql database create \
-            --search-path "${{ github.workspace }}/ruby/extractor-pack" \
-            --threads 4 \
-            --language ruby --source-root "${{ github.workspace }}/repo" \
-            "${{ runner.temp }}/database"
-      - name: Measure database
-        run: |
-          mkdir -p "stats/${{ matrix.repo }}"
-          codeql dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ruby"
-      - uses: actions/upload-artifact@v3
-        with:
-          name: measurements
-          path: stats
-          retention-days: 1
-
-  merge:
-    runs-on: ubuntu-latest
-    needs: measure
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
-        with:
-          name: measurements
-          path: stats
-      - run: |
-          python -m pip install --user lxml
-          find stats -name 'stats.xml' | sort | xargs python ruby/scripts/merge_stats.py --output ruby/ql/lib/ruby.dbscheme.stats --normalise ruby_tokeninfo
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ruby.dbscheme.stats
-          path: ruby/ql/lib/ruby.dbscheme.stats

.github/workflows/ruby-qltest.yml

@@ -1,67 +0,0 @@
-name: "Ruby: Run QL Tests"
-
-on:
-  push:
-    paths:
-      - "ruby/**"
-      - .github/workflows/ruby-build.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "ruby/**"
-      - .github/workflows/ruby-qltest.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-      - "rc/*"
-
-env:
-  CARGO_TERM_COLOR: always
-
-defaults:
-  run:
-    working-directory: ruby
-
-jobs:
-  qlupgrade:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - name: Check DB upgrade scripts
-        run: |
-          echo >empty.trap
-          codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
-          codeql dataset upgrade testdb --additional-packs ql/lib
-          diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme
-      - name: Check DB downgrade scripts
-        run: |
-          echo >empty.trap
-          rm -rf testdb; codeql dataset import -S ql/lib/ruby.dbscheme testdb empty.trap
-          codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
-           --dbscheme=ql/lib/ruby.dbscheme --target-dbscheme=downgrades/initial/ruby.dbscheme |
-           xargs codeql execute upgrades testdb
-          diff -q testdb/ruby.dbscheme downgrades/initial/ruby.dbscheme
-  qltest:
-    runs-on: ubuntu-latest-xl
-    strategy:
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - uses: ./ruby/actions/create-extractor-pack
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: ruby-qltest
-      - name: Run QL tests
-        run: |
-          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-        env:
-          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/swift.yml

@@ -1,103 +0,0 @@
-name: "Swift"
-
-on:
-  pull_request:
-    paths:
-      - "swift/**"
-      - "misc/bazel/**"
-      - "*.bazel*"
-      - .github/workflows/swift.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-      - .pre-commit-config.yaml
-      - "!**/*.md"
-      - "!**/*.qhelp"
-    branches:
-      - main
-      - rc/*
-  push:
-    paths:
-      - "swift/**"
-      - "misc/bazel/**"
-      - "*.bazel*"
-      - .github/workflows/swift.yml
-      - .github/actions/**
-      - codeql-workspace.yml
-      - "!**/*.md"
-      - "!**/*.qhelp"
-    branches:
-      - main
-      - rc/*
-
-jobs:
-  # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks
-  # without waiting for the macOS build
-  build-and-test-macos:
-    runs-on: macos-12-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/build-and-test
-  build-and-test-linux:
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/build-and-test
-  qltests-linux:
-    needs: build-and-test-linux
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/run-ql-tests
-  qltests-macos:
-    if : ${{ github.event_name == 'pull_request' }}
-    needs: build-and-test-macos
-    runs-on: macos-12-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/run-ql-tests
-  integration-tests-linux:
-    needs: build-and-test-linux
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/run-integration-tests
-  integration-tests-macos:
-    if : ${{ github.event_name == 'pull_request' }}
-    needs: build-and-test-macos
-    runs-on: macos-12-xl
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./swift/actions/run-integration-tests
-  codegen:
-    if : ${{ github.event_name == 'pull_request' }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: bazelbuild/setup-bazelisk@v2
-      - uses: actions/setup-python@v4
-        with:
-          python-version-file: 'swift/.python-version'
-      - uses: pre-commit/action@v3.0.0
-        name: Check that python code is properly formatted
-        with:
-          extra_args: autopep8 --all-files
-      - uses: ./.github/actions/fetch-codeql
-      - uses: pre-commit/action@v3.0.0
-        name: Check that QL generated code was checked in
-        with:
-          extra_args: swift-codegen --all-files
-      - name: Generate C++ files
-        run: |
-          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/generated-cpp-files
-      - uses: actions/upload-artifact@v3
-        with:
-          name: swift-generated-cpp-files
-          path: generated-cpp-files/**
-  database-upgrade-scripts:
-    if : ${{ github.event_name == 'pull_request' }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - uses: ./swift/actions/database-upgrade-scripts

.github/workflows/sync-files.yml

@@ -1,20 +0,0 @@
-name: Check synchronized files
-
-on:
-  push:
-    branches:
-      - main
-      - 'rc/*'
-  pull_request:
-    branches:
-      - main
-      - 'rc/*'
-
-jobs:
-  sync:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check synchronized files
-        run: python config/sync-files.py
-

.github/workflows/sync_files.yml

@@ -0,0 +1,21 @@
+name: Check synchronized files
+
+on:
+  push:
+    branches:
+      - main
+      - 'rc/*'
+  pull_request:
+    branches:
+      - main
+      - 'rc/*'
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Check synchronized files
+        run: scripts/sync-identical-files.py

.github/workflows/validate-change-notes.yml

@@ -1,31 +0,0 @@
-name: Validate change notes
-
-on:
-  push:
-    paths:
-      - "*/ql/*/change-notes/**/*"
-      - ".github/workflows/validate-change-notes.yml"
-      - ".github/actions/fetch-codeql/action.yml"
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "*/ql/*/change-notes/**/*"
-      - ".github/workflows/validate-change-notes.yml"
-      - ".github/actions/fetch-codeql/action.yml"
-
-jobs:
-  check-change-note:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Setup CodeQL
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Fail if there are any errors with existing change notes
-
-        run: |
-          codeql pack release --groups cpp,csharp,java,javascript,python,ruby,-examples,-test,-experimental

.gitignore

@@ -1,61 +1,8 @@
-# editor and OS artifacts
-*~
-.DS_STORE
-*.swp
-
-# query compilation caches
+/target
+extractor-pack
+.vscode/launch.json
 .cache
-
-# qltest projects and artifacts
-*/ql/test/**/*.testproj
-*/ql/test/**/*.actual
-*/ql/test/**/go.sum
-
-# Visual studio temporaries, except a file used by QL4VS
-.vs/*
-!.vs/VSWorkspaceSettings.json
-
-# Byte-compiled python files
-*.pyc
-
-# python virtual environment folder
-.venv/
-
-# binary files created by pytest-cov
-.coverage
-
-# It's useful (though not required) to be able to unpack codeql in the ql checkout itself
-/codeql/
-
-# Avoid committing cached package components
+ql/test/**/*.testproj
+ql/test/**/*.actual
+ql/test/**/CONSISTENCY
 .codeql
-
-# Compiled class file
-*.class
-
-# links created by bazel
-/bazel-*
-
-# local bazel options
-/local.bazelrc
-
-# CLion project files
-/.clwb
-
-# Go build artifacts
-go/build/*
-
-# Go binaries
-go/tools/bin
-go/tools/linux64
-go/tools/osx64
-go/tools/win64
-go/tools/tokenizer.jar
-go/main
-
-# node_modules folders except in the JS test suite
-node_modules/
-!/javascript/ql/test/**/node_modules/
-
-# Temporary folders for working with generated models
-.model-temp

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "codeql"]
+	path = codeql
+	url = https://github.com/github/codeql.git

.lgtm.yml

@@ -1,31 +0,0 @@
-path_classifiers:
-  library:
-  - javascript/externs
-  - javascript/extractor/lib
-
-  test:
-  - csharp/ql/src
-  - csharp/ql/test
-  - go/ql/test
-  - javascript/extractor/parser-tests
-  - javascript/extractor/tests
-  - javascript/ql/src
-  - javascript/ql/test
-  - python/ql/src
-  - python/ql/test
-
-  example:
-  - go/ql/src
-
-queries:
-  - include: "*"
-
-extraction:
-  python:
-    python_setup:
-      version: 3
-  javascript:
-    index:
-      exclude:
-        - javascript/ql/test
-        - javascript/extractor/tests

.pre-commit-config.yaml

@@ -1,57 +0,0 @@
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
-    hooks:
-      - id: trailing-whitespace
-        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
-      - id: end-of-file-fixer
-        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
-
-  - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v13.0.1
-    hooks:
-      - id: clang-format
-        files: ^swift/.*\.(h|c|cpp)$
-
-  - repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v1.6.0
-    hooks:
-      - id: autopep8
-        files: ^swift/.*\.py
-
-  - repo: local
-    hooks:
-      - id: codeql-format
-        name: Fix QL file formatting
-        files: \.qll?$
-        language: system
-        entry: codeql query format --in-place
-
-      - id: sync-files
-        name: Fix files required to be identical
-        files: \.(qll?|qhelp|swift)$|^config/identical-files\.json$
-        language: system
-        entry: python3 config/sync-files.py --latest
-        pass_filenames: false
-
-      - id: qhelp
-        name: Check query help generation
-        files: \.qhelp$
-        language: system
-        entry: python3 misc/scripts/check-qhelp.py
-
-      - id: swift-codegen
-        name: Run Swift checked in code generation
-        files: ^swift/(schema.py$|codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements)|ql/\.generated.list)
-        language: system
-        entry: bazel run //swift/codegen -- --quiet
-        pass_filenames: false
-
-      - id: swift-codegen-unit-tests
-        name: Run Swift code generation unit tests
-        files: ^swift/codegen/.*\.py$
-        language: system
-        entry: bazel test //swift/codegen/test
-        pass_filenames: false

.vscode/.gitattributes

@@ -1 +0,0 @@
-*.json  linguist-language=JSON-with-Comments

.vscode/extensions.json

@@ -1,10 +0,0 @@
-{
-    // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations.
-    // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp
-    // List of extensions which should be recommended for users of this workspace.
-    "recommendations": [
-        "GitHub.vscode-codeql"
-    ],
-    // List of extensions recommended by VS Code that should not be recommended for users of this workspace.
-    "unwantedRecommendations": []
-}

.vscode/settings.json

@@ -1,5 +0,0 @@
-{
-  "omnisharp.autoStart": false,
-  "cmake.sourceDirectory": "${workspaceFolder}/swift",
-  "cmake.buildDirectory": "${workspaceFolder}/bazel-cmake-build"
-}

.vscode/tasks.json

@@ -1,27 +1,14 @@
 {
-    // To run a task, select the `Terminal | Run Task...` menu option, and then select the task from
-    // the list in the dropdown, or invoke the `Tasks: Run Task` command from the command palette/
-    // To bind a keyboard shortcut to invoke a task, see https://code.visualstudio.com/docs/editor/tasks#_binding-keyboard-shortcuts-to-tasks.
-    // See https://go.microsoft.com/fwlink/?LinkId=733558
-    // for the documentation about the tasks.json format
-    "version": "2.0.0",
-    "tasks": [
-        {
-            "label": "Sync Identical Files",
-            "type": "process",
-            // Non-Windows OS will usually have Python 3 already installed at /usr/bin/python3.
-            "command": "python3",
-            "args": [
-                "config/sync-files.py",
-                "--latest"
-            ],
-            "group": "build",
-            "windows": {
-                // On Windows, use whatever Python interpreter is configured for this workspace. The default is
-                // just `python`, so if Python is already on the path, this will find it.
-                "command": "${config:python.pythonPath}",
-            },
-            "problemMatcher": []
-        }
-    ]
+	"version": "2.0.0",
+	"tasks": [
+		{
+			"type": "cargo",
+			"subcommand": "build",
+			"problemMatcher": [
+				"$rustc"
+			],
+			"group": "build",
+			"label": "Rust: cargo build"
+		}
+	]
 }

CODEOWNERS

@@ -1,41 +0,0 @@
-/cpp/ @github/codeql-c-analysis
-/csharp/ @github/codeql-csharp
-/go/ @github/codeql-go
-/java/ @github/codeql-java
-/javascript/ @github/codeql-javascript
-/python/ @github/codeql-python
-/ruby/ @github/codeql-ruby
-/swift/ @github/codeql-swift
-/java/kotlin-extractor/ @github/codeql-kotlin
-/java/kotlin-explorer/ @github/codeql-kotlin
-
-# ML-powered queries
-/javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
-
-# CodeQL tools and associated docs
-/docs/codeql/codeql-cli/ @github/codeql-cli-reviewers
-/docs/codeql/codeql-for-visual-studio-code/ @github/codeql-vscode-reviewers
-/docs/codeql/ql-language-reference/ @github/codeql-frontend-reviewers
-/docs/query-*-style-guide.md @github/codeql-analysis-reviewers
-
-# QL for QL reviewers
-/ql/ @github/codeql-ql-for-ql-reviewers
-
-# Bazel (excluding BUILD.bazel files)
-WORKSPACE.bazel @github/codeql-ci-reviewers
-.bazelversion @github/codeql-ci-reviewers
-.bazelrc @github/codeql-ci-reviewers
-**/*.bzl @github/codeql-ci-reviewers
-
-# Documentation etc
-/*.md @github/code-scanning-product
-/LICENSE @github/code-scanning-product
-
-# Workflows
-/.github/workflows/ @github/codeql-ci-reviewers
-/.github/workflows/atm-* @github/codeql-ml-powered-queries-reviewers
-/.github/workflows/go-* @github/codeql-go
-/.github/workflows/js-ml-tests.yml @github/codeql-ml-powered-queries-reviewers
-/.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
-/.github/workflows/ruby-* @github/codeql-ruby
-/.github/workflows/swift.yml @github/codeql-swift

CODE_OF_CONDUCT.md

@@ -1,126 +1,76 @@
+# Contributor Covenant Code of Conduct
+
 ## Our Pledge
 
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, religion, or sexual identity
-and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to make participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
 
 ## Our Standards
 
-Examples of behavior that contributes to a positive environment for our
-community include:
+Examples of behavior that contributes to creating a positive environment
+include:
 
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the
-  overall community
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
 
-Examples of unacceptable behavior include:
+Examples of unacceptable behavior by participants include:
 
-* The use of sexualized language or imagery, and sexual attention or
-  advances of any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
 * Public or private harassment
-* Publishing others' private information, such as a physical or email
-  address, without their explicit permission
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
 * Other conduct which could reasonably be considered inappropriate in a
   professional setting
 
-## Enforcement Responsibilities
+## Our Responsibilities
 
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
 
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
 
 ## Scope
 
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
 
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at
-opensource@github.com.
-All complaints will be reviewed and investigated promptly and fairly.
+reported by contacting the project team at opensource@github.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
 
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident.
-
-## Enforcement Guidelines
-
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series
-of actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or
-permanent ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior,  harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within
-the community.
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
 
 ## Attribution
 
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.0, available at
-https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
-
-Community Impact Guidelines were inspired by [Mozilla's code of conduct
-enforcement ladder](https://github.com/mozilla/diversity).
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
 
 [homepage]: https://www.contributor-covenant.org
 
-For answers to common questions about this code of conduct, see the FAQ at
-https://www.contributor-covenant.org/faq. Translations are available at
-https://www.contributor-covenant.org/translations.
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

CONTRIBUTING.md

@@ -1,77 +1,64 @@
-# Contributing to CodeQL
+## Contributing
 
-We welcome contributions to our CodeQL libraries and queries. Got an idea for a new check, or how to improve an existing query? Then please go ahead and open a pull request! Contributions to this project are [released](https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license) to the public under the [project's open source license](LICENSE).
+Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great.
 
-There is lots of useful documentation to help you write queries, ranging from information about query file structure to tutorials for specific target languages. For more information on the documentation available, see [CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/codeql-queries) on [codeql.github.com](https://codeql.github.com).
+Contributions to this project are [released](https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license) to the public under the [project's open source license](LICENSE).
 
-## Change notes
+Please note that this project is released with a [Contributor Code of Conduct](CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms.
 
-Any nontrivial user-visible change to a query pack or library pack should have a change note. For details on how to add a change note for your change, see [this guide](docs/change-notes.md).
+## Building and testing
 
-## Submitting a new experimental query
+See [Developer information](docs/HOWTO.md) for information on building the Ruby extractor. There is no need to rebuild the extractor if you are only developing queries.
 
-If you have an idea for a query that you would like to share with other CodeQL users, please open a pull request to add it to this repository. New queries start out in a `<language>/ql/src/experimental` directory, to which they can be merged when they meet the following requirements.
+1. Install the CodeQL CLI as described in [Getting started with the CodeQL CLI](https://codeql.github.com/docs/codeql-cli/getting-started-with-the-codeql-cli/).
 
-1. **Directory structure**
+2. Ensure that `<extraction-root>/codeql` is in your `PATH`.
 
-    There are six language-specific query directories in this repository:
+3. Clone this repository into `<extraction-root>/codeql-ruby` and change to this directory.
 
-      * C/C++: `cpp/ql/src`
-      * C#: `csharp/ql/src`
-      * Java: `java/ql/src`
-      * JavaScript: `javascript/ql/src`
-      * Python: `python/ql/src`
-      * Ruby: `ruby/ql/src`
+4. To run all tests in a directory and its subdirectories, run `codeql test run <directory>`, for example `codeql test run ql/test/query-tests/security`.
 
-    Each language-specific directory contains further subdirectories that group queries based on their `@tags` or purpose.
-    - Experimental queries and libraries are stored in the `experimental` subdirectory within each language-specific directory in the [CodeQL repository](https://github.com/github/codeql). For example, experimental Java queries and libraries are stored in `java/ql/src/experimental` and any corresponding tests in `java/ql/test/experimental`.
-    - Experimental queries need to include `experimental` in their `@tags`
-    - The structure of an `experimental` subdirectory mirrors the structure of its parent directory.
-    - Select or create an appropriate directory in `experimental` based on the existing directory structure of `experimental` or its parent directory.
+6. To run an individual test, run `codeql test run <filename>`, where `<filename>` is a `.ql` or `.qlref` file, for example `codeql test run ql/test/query-tests/security/cwe-078/CommandInjection.qlref`.
 
-2. **Query metadata**
+## Adding a new query
 
-    - The query `@id` must conform to all the requirements in the [guide on query metadata](docs/query-metadata-style-guide.md#query-id-id). In particular, it must not clash with any other queries in the repository, and it must start with the appropriate language-specific prefix.
-    - The query must have a `@name` and `@description` to explain its purpose.
-    - The query must have a `@kind` and `@problem.severity` as required by CodeQL tools.
+If you have an idea for a query that you would like to share with other CodeQL users, please open a pull request to add it to this repository.
+Follow the steps below to help other users understand what your query does, and to ensure that your query is consistent with the other CodeQL queries.
 
-    For details, see the [guide on query metadata](docs/query-metadata-style-guide.md).
+1. **Consult the documentation for query writers**
 
-    Make sure the `select` statement is compatible with the query `@kind`. See [About CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/about-codeql-queries/#select-clause) on codeql.github.com.
+   There is lots of useful documentation to help you write CodeQL queries, ranging from information about query file structure to language-specific tutorials. For more information on the documentation available, see [Writing CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/) and the [CodeQL documentation](https://codeql.github.com/docs).
 
-3. **Formatting**
+2. **Format your code correctly**
 
-    - The queries and libraries must be autoformatted, for example using the "Format Document" command in [CodeQL for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-codeql-for-visual-studio-code).
+   All of the standard CodeQL queries and libraries are uniformly formatted for clarity and consistency, so we strongly recommend that all contributions follow the same formatting guidelines. If you use the CodeQL extension for Visual Studio Code, you can auto-format your query using the [Format Document command](https://help.semmle.com/codeql/codeql-for-vscode/procedures/about-codeql-for-vscode.html). For more information, see the [QL style guide](https://github.com/github/codeql/blob/main/docs/ql-style-guide.md).
 
-    If you prefer, you can either:
-    1. install the [pre-commit framework](https://pre-commit.com/) and install the configured hooks on this repo via `pre-commit install`, or
-    2. use this [pre-commit hook](misc/scripts/pre-commit) that automatically checks whether your files are correctly formatted.
+3. **Make sure your query has the correct metadata**
 
-    See the [pre-commit hook installation guide](docs/pre-commit-hook-setup.md) for instructions on the two approaches.
+   Query metadata is used to identify your query and make sure the query results are displayed properly.
+   The most important metadata to include are the `@name`, `@description`, and the `@kind`.
+   Other metadata properties (`@precision`, `@severity`, and `@tags`) are usually added after the query has been reviewed by the maintainers.
+   For more information on writing query metadata, see the [Query metadata style guide](https://github.com/github/codeql/blob/main/docs/query-metadata-style-guide.md).
 
-4. **Compilation**
+4. **Make sure the `select` statement is compatible with the query type**
 
-    - Compilation of the query and any associated libraries and tests must be resilient to future development of the [supported](docs/supported-queries.md) libraries. This means that the functionality cannot use internal libraries, cannot depend on the output of `getAQlClass`, and cannot make use of regexp matching on `toString`.
-    - The query and any associated libraries and tests must not cause any compiler warnings to be emitted (such as use of deprecated functionality or missing `override` annotations).
+   The `select` statement of your query must be compatible with the query type (determined by the `@kind` metadata property) for alert or path results to be displayed correctly in LGTM and Visual Studio Code.
+   For more information on `select` statement format, see [About CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/about-codeql-queries/#select-clause) on the [CodeQL documentation](https://codeql.github.com/docs) site.
 
-5. **Results**
+5. **Write a query help file**
 
-    - The query must have at least one true positive result on some revision of a real project.
+   Query help files explain the purpose of your query to other users. Write your query help in a `.qhelp` file and save it in the same directory as your new query.
+   For more information on writing query help, see the [Query help style guide](https://github.com/github/codeql/blob/main/docs/query-help-style-guide.md).
 
-6. **Query help files and unit tests**
+6. **Maintain backwards compatibility**
 
-	- Query help (`.qhelp`) files and unit tests are optional (but strongly encouraged!) for queries in the `experimental` directories. For more information about contributing query help files and unit tests, see [Supported CodeQL queries and libraries](docs/supported-queries.md).
+The standard CodeQL libraries must evolve in a backwards compatible manner. If any backwards incompatible changes need to be made, the existing API must first be marked as deprecated. This is done by adding a `deprecated` annotation along with a QLDoc reference to the replacement API. Only after at least one full release cycle has elapsed may the old API be removed.
 
-Experimental queries and libraries may not be actively maintained as the supported libraries evolve. They may also be changed in backwards-incompatible ways or may be removed entirely in the future without deprecation warnings.
+In addition to contributions to our standard queries and libraries, we also welcome contributions of a more experimental nature, which do not need to fulfill all the requirements listed above. See the guidelines for [experimental queries and libraries](ql/docs/experimental.md) for details.
 
-After the experimental query is merged, we welcome pull requests to improve it. Before a query can be moved out of the `experimental` subdirectory, it must satisfy [the requirements for being a supported query](docs/supported-queries.md).
+## Resources
 
-## Using your personal data
-
-If you contribute to this project, we will record your name and email address (as provided by you with your contributions) as part of the code repositories, which are public. We might also use this information to contact you in relation to your contributions, as well as in the normal course of software development. We also store records of CLA agreements signed in the past, but no longer require contributors to sign a CLA. Under GDPR legislation, we do this on the basis of our legitimate interest in creating the CodeQL product.
-
-Please do get in touch (privacy@github.com) if you have any questions about this or our data protection policies.
-
-## Bazel
-Please notice that any bazel targets and definitions in this repository are currently experimental
-and for internal use only.
+- [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)
+- [Using Pull Requests](https://help.github.com/articles/about-pull-requests/)
+- [GitHub Help](https://help.github.com)
+- [A Note About Git Commit Messages](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)

Cargo.lock

@@ -0,0 +1,661 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "adler"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
+
+[[package]]
+name = "aho-corasick"
+version = "0.7.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "ansi_term"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "ansi_term"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
+
+[[package]]
+name = "bitflags"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+
+[[package]]
+name = "byteorder"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b"
+
+[[package]]
+name = "cc"
+version = "1.0.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "chrono"
+version = "0.4.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
+dependencies = [
+ "libc",
+ "num-integer",
+ "num-traits",
+ "time",
+ "winapi",
+]
+
+[[package]]
+name = "clap"
+version = "2.33.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002"
+dependencies = [
+ "ansi_term 0.11.0",
+ "atty",
+ "bitflags",
+ "strsim",
+ "textwrap",
+ "unicode-width",
+ "vec_map",
+]
+
+[[package]]
+name = "const_fn"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6"
+
+[[package]]
+name = "crc32fast"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
+dependencies = [
+ "cfg-if",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d"
+dependencies = [
+ "cfg-if",
+ "const_fn",
+ "crossbeam-utils",
+ "lazy_static",
+ "memoffset",
+ "scopeguard",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "lazy_static",
+]
+
+[[package]]
+name = "either"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
+
+[[package]]
+name = "flate2"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0"
+dependencies = [
+ "cfg-if",
+ "crc32fast",
+ "libc",
+ "miniz_oxide",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "itoa"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "libc"
+version = "0.2.85"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ccac4b00700875e6a07c6cde370d44d32fa01c5a65cdd2fca6858c479d28bb3"
+
+[[package]]
+name = "log"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "matchers"
+version = "0.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1"
+dependencies = [
+ "regex-automata",
+]
+
+[[package]]
+name = "memchr"
+version = "2.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
+
+[[package]]
+name = "memoffset"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "miniz_oxide"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
+dependencies = [
+ "adler",
+ "autocfg",
+]
+
+[[package]]
+name = "node-types"
+version = "0.1.0"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
+dependencies = [
+ "autocfg",
+ "num-traits",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
+dependencies = [
+ "unicode-xid",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rayon"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
+dependencies = [
+ "autocfg",
+ "crossbeam-deque",
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
+ "lazy_static",
+ "num_cpus",
+]
+
+[[package]]
+name = "regex"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+ "thread_local",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4"
+dependencies = [
+ "byteorder",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581"
+
+[[package]]
+name = "ruby-autobuilder"
+version = "0.1.0"
+
+[[package]]
+name = "ruby-extractor"
+version = "0.1.0"
+dependencies = [
+ "clap",
+ "flate2",
+ "node-types",
+ "num_cpus",
+ "rayon",
+ "regex",
+ "tracing",
+ "tracing-subscriber",
+ "tree-sitter",
+ "tree-sitter-embedded-template",
+ "tree-sitter-ruby",
+]
+
+[[package]]
+name = "ruby-generator"
+version = "0.1.0"
+dependencies = [
+ "clap",
+ "node-types",
+ "tracing",
+ "tracing-subscriber",
+ "tree-sitter-embedded-template",
+ "tree-sitter-ruby",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
+
+[[package]]
+name = "scopeguard"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+
+[[package]]
+name = "serde"
+version = "1.0.123"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.123"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.61"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "sharded-slab"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79c719719ee05df97490f80a45acfc99e5a30ce98a1e4fb67aee422745ae14e3"
+dependencies = [
+ "lazy_static",
+]
+
+[[package]]
+name = "smallvec"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
+
+[[package]]
+name = "strsim"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
+
+[[package]]
+name = "syn"
+version = "1.0.60"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
+dependencies = [
+ "unicode-width",
+]
+
+[[package]]
+name = "thread_local"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
+name = "time"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
+dependencies = [
+ "libc",
+ "wasi",
+ "winapi",
+]
+
+[[package]]
+name = "tracing"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7d40a22fd029e33300d8d89a5cc8ffce18bb7c587662f54629e94c9de5487f3"
+dependencies = [
+ "cfg-if",
+ "pin-project-lite",
+ "tracing-attributes",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-attributes"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43f080ea7e4107844ef4766459426fa2d5c1ada2e47edba05dc7fa99d9629f47"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f"
+dependencies = [
+ "lazy_static",
+]
+
+[[package]]
+name = "tracing-log"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e0f8c7178e13481ff6765bd169b33e8d554c5d2bbede5e32c356194be02b9b9"
+dependencies = [
+ "lazy_static",
+ "log",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-serde"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b"
+dependencies = [
+ "serde",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-subscriber"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1fa8f0c8f4c594e4fc9debc1990deab13238077271ba84dd853d54902ee3401"
+dependencies = [
+ "ansi_term 0.12.1",
+ "chrono",
+ "lazy_static",
+ "matchers",
+ "regex",
+ "serde",
+ "serde_json",
+ "sharded-slab",
+ "smallvec",
+ "thread_local",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
+ "tracing-serde",
+]
+
+[[package]]
+name = "tree-sitter"
+version = "0.19.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad726ec26496bf4c083fff0f43d4eb3a2ad1bba305323af5ff91383c0b6ecac0"
+dependencies = [
+ "cc",
+ "regex",
+]
+
+[[package]]
+name = "tree-sitter-embedded-template"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "193fe1e3b39f71785d1890e4ac7bedd9be3d797f5db6363884415804fd516d93"
+dependencies = [
+ "cc",
+ "tree-sitter",
+]
+
+[[package]]
+name = "tree-sitter-ruby"
+version = "0.19.0"
+source = "git+https://github.com/tree-sitter/tree-sitter-ruby.git?rev=bb6a42e42b048627a74a127d3e0184c1eef01de9#bb6a42e42b048627a74a127d3e0184c1eef01de9"
+dependencies = [
+ "cc",
+ "tree-sitter",
+]
+
+[[package]]
+name = "unicode-width"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
+
+[[package]]
+name = "vec_map"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
+
+[[package]]
+name = "wasi"
+version = "0.10.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2006-2020 GitHub, Inc.
+Copyright (c) 2020-2021 GitHub
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -0,0 +1,71 @@
+all: extractor dbscheme
+
+ifeq ($(OS),Windows_NT)
+EXE = .exe
+CODEQL_PLATFORM = win64
+else
+EXE =
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Linux)
+CODEQL_PLATFORM = linux64
+endif
+ifeq ($(UNAME_S),Darwin)
+CODEQL_PLATFORM = osx64
+endif
+endif
+
+FILES=codeql-extractor.yml\
+      tools/qltest.cmd\
+      tools/index-files.sh\
+      tools/index-files.cmd\
+      tools/autobuild.sh\
+      tools/qltest.sh\
+      tools/autobuild.cmd\
+      ql/lib/ruby.dbscheme.stats\
+      ql/lib/ruby.dbscheme
+
+BIN_FILES=target/release/ruby-extractor$(EXE) target/release/ruby-autobuilder$(EXE)
+
+extractor-common:
+	rm -rf build
+	mkdir build
+	mkdir build/codeql-extractor-ruby
+	cp codeql-extractor.yml ql/lib/ruby.dbscheme ql/lib/ruby.dbscheme.stats build/codeql-extractor-ruby
+	cp -r tools build/codeql-extractor-ruby/
+
+.PHONY:	tools
+tools: $(BIN_FILES)
+	rm -rf tools/bin
+	mkdir tools/bin
+	cp -r target/release/ruby-autobuilder$(EXE) tools/bin/autobuilder$(EXE)
+	cp -r target/release/ruby-extractor$(EXE) tools/bin/extractor$(EXE)	
+
+target/release/%$(EXE):
+	cargo build --release --bin $(basename $(notdir $@))
+
+dbscheme:
+	cargo build --bin ruby-generator
+	cargo run -p ruby-generator -- --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+	codeql query format -i ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+
+.PHONY:	extractor
+extractor:	$(FILES) $(BIN_FILES)
+	rm -rf extractor-pack
+	mkdir extractor-pack
+	mkdir extractor-pack/tools
+	mkdir extractor-pack/tools/$(CODEQL_PLATFORM)
+	cp codeql-extractor.yml extractor-pack/codeql-extractor.yml
+	cp tools/qltest.cmd extractor-pack/tools/qltest.cmd
+	cp tools/index-files.sh extractor-pack/tools/index-files.sh
+	cp tools/index-files.cmd extractor-pack/tools/index-files.cmd
+	cp tools/autobuild.sh extractor-pack/tools/autobuild.sh
+	cp tools/qltest.sh extractor-pack/tools/qltest.sh
+	cp tools/autobuild.cmd extractor-pack/tools/autobuild.cmd
+	cp ql/lib/ruby.dbscheme.stats extractor-pack/ruby.dbscheme.stats
+	cp ql/lib/ruby.dbscheme extractor-pack/ruby.dbscheme
+	cp target/release/ruby-extractor$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/extractor$(EXE)
+	cp target/release/ruby-autobuilder$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/autobuilder$(EXE)
+
+test: extractor dbscheme
+	codeql pack install ql/test
+	codeql test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path . --consistency-queries ql/consistency-queries ql/test

README.md

@@ -1,31 +1,50 @@
-# CodeQL
+# Ruby analysis support for CodeQL
 
-This open source repository contains the standard CodeQL libraries and queries that power [GitHub Advanced Security](https://github.com/features/security/code) and the other application security products that [GitHub](https://github.com/features/security/) makes available to its customers worldwide.
+This open-source repository contains the extractor, CodeQL libraries, and queries that power Ruby
+support in [LGTM](https://lgtm.com) and the other CodeQL products that [GitHub](https://github.com)
+makes available to its customers worldwide.
 
-## How do I learn CodeQL and run queries?
+It contains two major components:
+  - an extractor, written in Rust, that parses Ruby source code and converts it into a database
+    that can be queried using CodeQL.
+  - static analysis libraries and queries written in [CodeQL](https://codeql.github.com/docs/) that can be
+    used to analyze such a database to find coding mistakes or security vulnerabilities.
 
-There is [extensive documentation](https://codeql.github.com/docs/) on getting started with writing CodeQL using the [CodeQL extension for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/) and the [CodeQL CLI](https://codeql.github.com/docs/codeql-cli/).
+The goal of this project is to provide comprehensive static analysis support for Ruby in CodeQL.
 
-## Contributing
+For the queries and libraries that power CodeQL support for other languages, visit [the CodeQL
+repository](https://github.com/github/codeql).
 
-We welcome contributions to our standard library and standard checks. Do you have an idea for a new check, or how to improve an existing query? Then please go ahead and open a pull request! Before you do, though, please take the time to read our [contributing guidelines](CONTRIBUTING.md). You can also consult our [style guides](https://github.com/github/codeql/tree/main/docs) to learn how to format your code for consistency and clarity, how to write query metadata, and how to write query help documentation for your query.
+## Installation
 
-For information on contributing to CodeQL documentation, see the "[contributing guide](docs/codeql/CONTRIBUTING.md)" for docs.
+Simply clone this repository. There are no external dependencies.
 
-## License
+If you want to use the CodeQL extension for Visual Studio Code, import this repository into your VS
+Code workspace.
 
-The code in this repository is licensed under the [MIT License](LICENSE) by [GitHub](https://github.com).
+## Usage
 
-The CodeQL CLI (including the CodeQL engine) is hosted in a [different repository](https://github.com/github/codeql-cli-binaries) and is [licensed separately](https://github.com/github/codeql-cli-binaries/blob/main/LICENSE.md). If you'd like to use the CodeQL CLI to analyze closed-source code, you will need a separate commercial license; please [contact us](https://github.com/enterprise/contact) for further help.
+To analyze a Ruby codebase, either use the [CodeQL command-line
+interface](https://codeql.github.com/docs/codeql-cli/) to create a database yourself, or
+download a pre-built database from [LGTM.com](https://lgtm.com/). You can then run any of the
+queries contained in this repository either on the command line or using the VS Code extension.
 
-## Visual Studio Code integration
+Note that the [lgtm.com](https://github.com/github/codeql-ruby/tree/lgtm.com) branch of this
+repository corresponds to the version of the queries that is currently deployed on LGTM.com.
+The [main](https://github.com/github/codeql-ruby/tree/main) branch may contain changes that
+have not been deployed yet, so you may need to upgrade databases downloaded from [LGTM.com](https://lgtm.com) before
+running queries on them.
 
-If you use Visual Studio Code to work in this repository, there are a few integration features to make development easier.
+## Contributions
 
-### CodeQL for Visual Studio Code
+Contributions are welcome! Please see our [contribution guidelines](CONTRIBUTING.md) and our
+[code of conduct](CODE_OF_CONDUCT.md) for details on how to participate in our community.
 
-You can install the [CodeQL for Visual Studio Code](https://marketplace.visualstudio.com/items?itemName=GitHub.vscode-codeql) extension to get syntax highlighting, IntelliSense, and code navigation for the QL language, as well as unit test support for testing CodeQL libraries and queries.
+## Licensing
 
-### Tasks
+The code in this repository is licensed under the [MIT license](LICENSE).
 
-The `.vscode/tasks.json` file defines custom tasks specific to working in this repository. To invoke one of these tasks, select the `Terminal | Run Task...` menu option, and then select the desired task from the dropdown. You can also invoke the `Tasks: Run Task` command from the command palette.
+## Resources
+
+- [Writing CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/)
+- [CodeQL documentation](https://codeql.github.com/docs/)

WORKSPACE.bazel

@@ -1,12 +0,0 @@
-# Please notice that any bazel targets and definitions in this repository are currently experimental
-# and for internal use only.
-
-workspace(name = "codeql")
-
-load("//misc/bazel:workspace.bzl", "codeql_workspace")
-
-codeql_workspace()
-
-load("//misc/bazel:workspace_deps.bzl", "codeql_workspace_deps")
-
-codeql_workspace_deps()

autobuilder/src/main.rs

@@ -0,0 +1,39 @@
+use std::env;
+use std::path::PathBuf;
+use std::process::Command;
+
+fn main() -> std::io::Result<()> {
+    let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set");
+    let db = env::var("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE")
+        .expect("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE not set");
+    let codeql = if env::consts::OS == "windows" {
+        "codeql.exe"
+    } else {
+        "codeql"
+    };
+    let codeql: PathBuf = [&dist, codeql].iter().collect();
+    let mut cmd = Command::new(codeql);
+    cmd.arg("database")
+        .arg("index-files")
+        .arg("--include-extension=.rb")
+        .arg("--include-extension=.erb")
+        .arg("--include-extension=.gemspec")
+        .arg("--include=**/Gemfile")
+        .arg("--size-limit=5m")
+        .arg("--language=ruby")
+        .arg("--working-dir=.")
+        .arg(db);
+
+    for line in env::var("LGTM_INDEX_FILTERS")
+        .unwrap_or_default()
+        .split('\n')
+    {
+        if let Some(stripped) = line.strip_prefix("include:") {
+            cmd.arg("--include").arg(stripped);
+        } else if let Some(stripped) = line.strip_prefix("exclude:") {
+            cmd.arg("--exclude").arg(stripped);
+        }
+    }
+    let exit = &cmd.spawn()?.wait()?;
+    std::process::exit(exit.code().unwrap_or(1))
+}

change-notes/1.18/analysis-cpp.md

@@ -1,28 +0,0 @@
-# Improvements to C/C++ analysis
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Upcast array used in pointer arithmetic (`cpp/upcast-array-pointer-arithmetic`) | reliability, correctness, external/cwe/cwe-119 | Finds undefined behavior caused by doing pointer arithmetic on an array of objects that has been cast to an array of a supertype. |
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Assignment where comparison was intended (`cpp/assign-where-compare-meant`) | Fewer false positive results | Results are no longer reported if the variable is not yet defined. |
-| Comparison where assignment was intended (`cpp/compare-where-assign-meant`) | More results | This query now includes results where an overloaded `operator==` is used in the wrong context. |
-| For loop variable changed in body (`cpp/loop-variable-changed`)  | Fewer false positive results | Results where the loop variable is a member of a class or struct now account for the object. |
-| Local variable hides global variable (`cpp/local-variable-hides-global-variable`) | Fewer false positive results | Results for parameters are now only reported if the name of the global variable is the same as the name of the parameter as used in the function definition (not just a function declaration). |
-| Nested loops with same variable (`cpp/nested-loops-with-same-variable`) | Fewer false positive results | Results where the loop variable is a member of a class or struct now account for the object. |
-| Self comparison (`cpp/comparison-of-identical-expressions`) | Fewer false positive results | Range checks of the form `x == (T)x` are no longer flagged unless they are guaranteed to have the same result on all platforms. |
-| Too few arguments to formatting function (`cpp/wrong-number-format-arguments`) | More precise results | This was previously known as "Wrong number of arguments to formatting function". It now focuses only on functions calls that are missing arguments, which tend to be more severe. See the next row for the new query that reports lower-severity alerts for calls with too many arguments. In addition, both queries now understand positional format arguments as supported by some libraries, and some false positive results for custom printf-like functions have been fixed.|
-| Too many arguments to formatting function (`cpp/too-many-format-arguments`) | More precise results | This new query was created by splitting the old "Wrong number of arguments to formatting function" query (see row above). It reports function calls with too many arguments.  |
-| User-controlled data in arithmetic expression (`cpp/tainted-arithmetic`) | More results | The query is extended to analyze increment, decrement, addition-assignment, and subtraction-assignment operations. |
-| Variable used in its own initializer (`cpp/use-in-own-initializer`) | Fewer false positive results | Results where a macro is used to indicate deliberate uninitialization are now excluded. |
-|Uncontrolled data in arithmetic expression (`cpp/uncontrolled-arithmetic`) | More results | The query is extended to analyze increment, decrement, addition-assignment, and subtraction-assignment operations. |
- 
-## Changes to QL libraries
-
-* The `ClassAggregateLiteral.getFieldExpr()` and `ArrayAggregateLiteral.getElementExpr()` predicates incorrectly assumed that initializer expressions appeared in the same order as the declaration order of the elements. This resulted in the association of the expressions with the wrong elements when designated initializers were used. This has been fixed.
-* Results for the `Element.getEnclosingElement()` predicate no longer included macro accesses. To explore parents and children of macro accesses, use the relevant member predicates on `MacroAccess` or `MacroInvocation`.

change-notes/1.18/analysis-csharp.md

@@ -1,60 +0,0 @@
-# Improvements to C# analysis
-
-## General improvements
-
-* Control flow analysis has been improved for `catch` clauses with filters.
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Arbitrary file write during zip extraction ("Zip Slip") (`cs/zipslip`) | security, external/cwe/cwe-022  | Identifies zip extraction routines which allow arbitrary file overwrite vulnerabilities. |
-| Local scope variable shadows member (`cs/local-shadows-member`) | maintainability, readability | Replaces the existing queries Local variable shadows class member (`cs/local-shadows-class-member`), Local variable shadows struct member (`cs/local-shadows-struct-member`), Parameter shadows class member (`cs/parameter-shadows-class-member`), and Parameter shadows struct member (`cs/parameter-shadows-struct-member`). |
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Constant condition (`cs/constant-condition`) | More results | The query has been generalized to report alerts for the old queries Null-coalescing left operand is constant (`cs/constant-null-coalescing`) and Switch selector is constant (`cs/constant-switch-selector`). |
-| Exposing internal representation (`cs/expose-implementation`) | Different results | The query has been rewritten, based on the [equivalent Java query](https://help.semmle.com/wiki/display/JAVA/Exposing+internal+representation). |
-| Local variable shadows class member (`cs/local-shadows-class-member`) | No results | The query has been replaced by the new query: Local scope variable shadows member  (`cs/local-shadows-member`). |
-| Local variable shadows struct member (`cs/local-shadows-struct-member`) | No results | The query has been replaced by the new query: Local scope variable shadows member  (`cs/local-shadows-member`). |
-| Missing Dispose call on local IDisposable (`cs/local-not-disposed`) | Fewer false positive results | The query identifies more cases where the local variable may be disposed by a library call. |
-| Nested loops with same variable (`cs/nested-loops-with-same-variable`) | Fewer false positive results | Results are no longer highlighted in nested loops that share the same condition, and do not use the variable after the inner loop. |
-| Null-coalescing left operand is constant (`cs/constant-null-coalescing`) | No results | The query has been removed, as alerts for this problem are now reported by the new query: Constant condition (`cs/constant-condition`). |
-| Parameter shadows class member (`cs/parameter-shadows-class-member`) | No results | The query has been replaced by the new query: Local scope variable shadows member  (`cs/local-shadows-member`). |
-| Parameter shadows struct member (`cs/parameter-shadows-struct-member`) | No results | The query has been replaced by the new query: Local scope variable shadows member  (`cs/local-shadows-member`). |
-| Potentially incorrect CompareTo(...) signature (`cs/wrong-compareto-signature`) | Fewer false positive results | Results are no longer highlighted in constructed types. |
-| Switch selector is constant (`cs/constant-switch-selector`) | No results | The query has been removed, as alerts for this problem are now reported by the new query: Constant condition (`cs/constant-condition`). |
-| Useless upcast (`cs/useless-upcast`) | Fewer false positive results | The query has been improved to cover more cases where upcasts may be needed. |
-
-## Changes to code extraction
-
-* The `into` part of `join` clauses is now extracted.
-* The `when` part of constant cases is now extracted.
-* Fixed a bug where `while(x is T y) ...` was not extracted correctly.
-
-## Changes to QL libraries
-
-* A new non-member predicate `mayBeDisposed()` can be used to determine if a variable is potentially disposed inside a library. It will analyze the CIL code in the library to determine this.
-* The predicate `getCondition()` has been moved from `TypeCase` to `CaseStmt`. It is now possible to get the condition of a `ConstCase` using its `getCondition()` predicate.
-* Several control flow graph entities have been renamed (the old names are deprecated but are still available in this release for backwards compatibility):
-  - `ControlFlowNode` has been renamed to `ControlFlow::Node`.
-  - `CallableEntryNode` has been renamed to `ControlFlow::Nodes::EntryNode`.
-  - `CallableExitNode` has been renamed to `ControlFlow::Nodes::ExitNode`.
-  - `ControlFlowEdgeType` has been renamed to `ControlFlow::SuccessorType`.
-  - `ControlFlowEdgeSuccessor` has been renamed to `ControlFlow::SuccessorTypes::NormalSuccessor`.
-  - `ControlFlowEdgeConditional` has been renamed to `ControlFlow::SuccessorTypes::ConditionalSuccessor`.
-  - `ControlFlowEdgeBoolean` has been renamed to `ControlFlow::SuccessorTypes::BooleanSuccessor`.
-  - `ControlFlowEdgeNullness` has been renamed to `ControlFlow::SuccessorTypes::NullnessSuccessor`.
-  - `ControlFlowEdgeMatching` has been renamed to `ControlFlow::SuccessorTypes::MatchingSuccessor`.
-  - `ControlFlowEdgeEmptiness` has been renamed to `ControlFlow::SuccessorTypes::EmptinessSuccessor`.
-  - `ControlFlowEdgeReturn` has been renamed to `ControlFlow::SuccessorTypes::ReturnSuccessor`.
-  - `ControlFlowEdgeBreak` has been renamed to `ControlFlow::SuccessorTypes::BreakSuccessor`.
-  - `ControlFlowEdgeContinue` has been renamed to `ControlFlow::SuccessorTypes::ContinueSuccessor`.
-  - `ControlFlowEdgeGotoLabel` has been renamed to `ControlFlow::SuccessorTypes::GotoLabelSuccessor`.
-  - `ControlFlowEdgeGotoCase` has been renamed to `ControlFlow::SuccessorTypes::GotoCaseSuccessor`.
-  - `ControlFlowEdgeGotoDefault` has been renamed to `ControlFlow::SuccessorTypes::GotoDefaultSuccessor`.
-  - `ControlFlowEdgeException` has been renamed to `ControlFlow::SuccessorTypes::ExceptionSuccessor`.
-
-> You should update any custom queries that use these entities to ensure that they continue working when the old names are removed in a future release.

change-notes/1.18/analysis-javascript.md

@@ -1,139 +0,0 @@
-# Improvements to JavaScript analysis
-
-## General improvements
-
-* Improved modeling of data flow through destructuring assignments may give additional results for the security queries and other queries that rely on data flow.
-
-* Improved modeling of global variables may give more true-positive results and fewer false-positive results for a variety of queries.
-
-* Improved modeling of re-export declarations may result in fewer false-positive results for a variety of queries.
-
-* Improved modeling of taint flow through array operations may give additional results for the security queries.
-
-* The taint tracking library recognizes more ways in which taint propagates. In particular, some flow through string formatters is now recognized. This may give additional results for the security queries.
-
-* The taint tracking library now recognizes additional sanitization patterns. This may give fewer false-positive results for the security queries.
-
-* Type inference for simple function calls has been improved. This may give additional results for queries that rely on type inference.
-
-* Additional heuristics have been added to `semmle.javascript.heuristics`. Add `import semmle.javascript.heuristics.all` to a query in order to activate all of the heuristics at once.
-
-* Handling of ambient TypeScript code has been improved. As a result, fewer false-positive results will be reported in `.d.ts` files.
-
-* Support for popular libraries has been improved. Consequently, queries may produce more results on code bases that use the following libraries:
- [axios](https://github.com/axios/axios), 
- [bluebird](https://bluebirdjs.com), 
- [browserid-crypto](https://github.com/mozilla/browserid-crypto), 
- [compose-function](https://github.com/stoeffel/compose-function), 
- [cookie-parser](https://github.com/expressjs/cookie-parser), 
- [cookie-session](https://github.com/expressjs/cookie-session), 
- [cross-fetch](https://github.com/lquixada/cross-fetch), 
- [crypto-js](https://github.com/https://github.com/brix/crypto-js), 
- [deep-assign](https://github.com/sindresorhus/deep-assign), 
- [deep-extend](https://github.com/unclechu/node-deep-extend), 
- [deep-merge](https://github.com/Raynos/deep-merge), 
- [deep](https://github.com/jeffomatic/deep), 
- [deepmerge](https://github.com/KyleAMathews/deepmerge), 
- [defaults-deep](https://github.com/jonschlinkert/defaults-deep), 
- [defaults](https://github.com/tmpvar/defaults), 
- [dottie](https://github.com/mickhansen/dottie.js), 
- [dotty](https://github.com/deoxxa/dotty), 
- [ent](https://github.com/substack/node-ent), 
- [entities](https://github.com/fb55/node-entities), 
- [escape-goat](https://github.com/sindresorhus/escape-goat), 
- [express-jwt](https://github.com/auth0/express-jwt), 
- [express-session](https://github.com/expressjs/session), 
- [extend-shallow](https://github.com/jonschlinkert/extend-shallow), 
- [extend](https://github.com/justmoon/node-extend), 
- [extend2](https://github.com/eggjs/extend2), 
- [fast-json-parse](https://github.com/mcollina/fast-json-parse), 
- [forge](https://github.com/digitalbazaar/forge), 
- [format-util](https://github.com/tmpfs/format-util), 
- [got](https://github.com/sindresorhus/got), 
- [global](https://github.com/Raynos/global), 
- [he](https://github.com/mathiasbynens/he), 
- [html-entities](https://github.com/mdevils/node-html-entities), 
- [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch), 
- [jquery](https://jquery.com), 
- [js-extend](https://github.com/vmattos/js-extend), 
- [json-parse-better-errors](https://github.com/zkat/json-parse-better-errors), 
- [json-parse-safe](https://github.com/joaquimserafim/json-parse-safe), 
- [json-safe-parse](https://github.com/bahamas10/node-json-safe-parse), 
- [just-compose](https://github.com/angus-c/just), 
- [just-extend](https://github.com/angus-c/just), 
- [lodash](https://lodash.com), 
- [merge-deep](https://github.com/jonschlinkert/merge-deep), 
- [merge-options](https://github.com/schnittstabil/merge-options), 
- [merge](https://github.com/yeikos/js.merge), 
- [mixin-deep](https://github.com/jonschlinkert/mixin-deep), 
- [mixin-object](https://github.com/jonschlinkert/mixin-object), 
- [MySQL2](https://github.com/sidorares/node-mysql2), 
- [node.extend](https://github.com/dreamerslab/node.extend), 
- [node-fetch](https://github.com/bitinn/node-fetch), 
- [object-assign](https://github.com/sindresorhus/object-assign), 
- [object.assign](https://github.com/ljharb/object.assign), 
- [object.defaults](https://github.com/jonschlinkert/object.defaults), 
- [parse-json](https://github.com/sindresorhus/parse-json), 
- [printf](https://github.com/adaltas/node-printf), 
- [printj](https://github.com/SheetJS/printj), 
- [q](https://documentup.com/kriskowal/q/), 
- [ramda](https://ramdajs.com), 
- [request](https://github.com/request/request), 
- [request-promise](https://github.com/request/request-promise), 
- [request-promise-any](https://github.com/request/request-promise-any), 
- [request-promise-native](https://github.com/request/request-promise-native), 
- [React Native](https://facebook.github.io/react-native/), 
- [safe-json-parse](https://github.com/Raynos/safe-json-parse), 
- [sanitize](https://github.com/pocketly/node-sanitize), 
- [sanitizer](https://github.com/theSmaw/Caja-HTML-Sanitizer), 
- [smart-extend](https://github.com/danielkalen/smart-extend), 
- [sprintf.js](https://github.com/alexei/sprintf.js), 
- [string-template](https://github.com/Matt-Esch/string-template), 
- [superagent](https://github.com/visionmedia/superagent), 
- [underscore](https://underscorejs.org), 
- [util-extend](https://github.com/isaacs/util-extend), 
- [utils-merge](https://github.com/jaredhanson/utils-merge), 
- [validator](https://github.com/chriso/validator.js), 
- [xss](https://github.com/leizongmin/js-xss), 
- [xtend](https://github.com/Raynos/xtend).
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Clear-text logging of sensitive information (`js/clear-text-logging`) | security, external/cwe/cwe-312, external/cwe/cwe-315, external/cwe/cwe-359 | Highlights logging of sensitive information, indicating a violation of [CWE-312](https://cwe.mitre.org/data/definitions/312.html). Results shown on LGTM by default. |
-| Disabling Electron webSecurity (`js/disabling-electron-websecurity`) | security, frameworks/electron | Highlights Electron browser objects that are created with the `webSecurity` property set to false. Results shown on LGTM by default. |
-| Enabling Electron allowRunningInsecureContent (`js/enabling-electron-insecure-content`) | security, frameworks/electron | Highlights Electron browser objects that are created with the `allowRunningInsecureContent` property set to true. Results shown on LGTM by default. |
-| Uncontrolled data used in remote request (`js/request-forgery`) | security, external/cwe/cwe-918 | Highlights remote requests that are built from unsanitized user input, indicating a violation of [CWE-918](https://cwe.mitre.org/data/definitions/918.html). Results are hidden on LGTM by default. |
-| Use of externally-controlled format string (`js/tainted-format-string`) | security, external/cwe/cwe-134 | Highlights format strings containing user-provided data, indicating a violation of [CWE-134](https://cwe.mitre.org/data/definitions/134.html). Results shown on LGTM by default. |
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Arguments redefined (`js/arguments-redefinition`) | Fewer results | This query previously also flagged redefinitions of `eval`. This was an oversight that is now fixed. |
-| Comparison between inconvertible types (`js/comparison-between-incompatible-types`) | Fewer results | This query now flags fewer comparisons involving parameters. The severity of this query has been revised to "warning". |
-| CORS misconfiguration for credentials transfer (`js/cors-misconfiguration-for-credentials`) | More true-positive results | This query now treats header names case-insensitively. |
-| Hard-coded credentials (`js/hardcoded-credentials`) | More true-positive results | This query now recognizes secret cryptographic keys. |
-| Incomplete string escaping or encoding (`js/incomplete-sanitization`) | New name, more true-positive results | The "Incomplete sanitization" query has been renamed to more clearly reflect its purpose. It now recognizes incomplete URL encoding and decoding. |
-| Insecure randomness (`js/insecure-randomness`) | More true-positive results | This query now recognizes secret cryptographic keys. |
-| Misleading indentation after control statement (`js/misleading-indentation-after-control-statement`) | Fewer results | This query temporarily ignores TypeScript files. |
-| Missing rate limiting (`js/missing-rate-limiting`) | More true-positive results, fewer false-positive results | This query now recognizes additional rate limiters and expensive route handlers. |
-| Omitted array element (`js/omitted-array-element`)| Fewer results | This query temporarily ignores TypeScript files. |
-| Reflected cross-site scripting (`js/reflected-xss`) | Fewer false-positive results | This query now treats header names case-insensitively. |
-| Semicolon insertion (`js/automatic-semicolon-insertion`) | Fewer results | This query temporarily ignores TypeScript files. |
-| Server-side URL redirect (`js/server-side-unvalidated-url-redirection`) | More true-positive results | This query now treats header names case-insensitively. |
-| Superfluous trailing arguments (`js/superfluous-trailing-arguments`) | Fewer false-positive results | This query now ignores calls to some empty functions. |
-| Type confusion through parameter tampering (`js/type-confusion-through-parameter-tampering`) | Fewer false-positive results | This query no longer flags emptiness checks. |
-| Uncontrolled command line (`js/command-line-injection`) | More true-positive results | This query now recognizes indirect command injection through `sh -c` and similar. |
-| Unused variable, import, function or class (`js/unused-local-variable`) | New name, fewer results | The "Unused variable" query has been renamed to reflect the fact that it highlights different kinds of unused program elements. In addition, the query no longer highlights class expressions that could be made anonymous. While technically true, these results are not interesting. |
-| Use of incompletely initialized object (`js/incomplete-object-initialization`) | Fewer results | This query now highlights the constructor instead of its erroneous `this` or `super` expressions. |
-| Useless conditional (`js/trivial-conditional`) | Fewer results | This query no longer flags uses of boolean return values and highlights fewer comparisons involving parameters. |
-
-## Changes to QL libraries
-
-* HTTP and HTTPS requests made using the Node.js `http.request` and `https.request` APIs, and the Electron `Electron.net.request` and `Electron.ClientRequest` APIs, are modeled as `RemoteFlowSources`.
-* HTTP header names are now always normalized to lower case to reflect the fact that they are case insensitive. In particular, the result of `HeaderDefinition.getAHeaderName`, and the first parameter of `HeaderDefinition.defines`, `ExplicitHeaderDefinition.definesExplicitly`, and `RouteHandler.getAResponseHeader` are now always a lower-case string.
-* New AST nodes have been added for TypeScript 2.9 and 3.0 features.
-* The class `JsonParseCall` has been deprecated. Update your queries to use `JsonParserCall` instead.
-* The handling of spread arguments in the data flow library has been changed: `DataFlow::InvokeNode.getArgument(i)` is now only defined when there is no spread argument at or before argument position `i`, and similarly `InvokeNode.getNumArgument` is only defined for invocations without spread arguments.

change-notes/1.19/analysis-cpp.md

@@ -1,53 +0,0 @@
-# Improvements to C/C++ analysis
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Cast between `HRESULT` and a Boolean type (`cpp/hresult-boolean-conversion`) | security, external/cwe/cwe-253 | Finds logic errors caused by mistakenly treating the Windows `HRESULT` type as a Boolean instead of testing it with the appropriate macros. Results are shown on LGTM by default. |
-| Cast from `char*` to `wchar_t*` (`cpp/incorrect-string-type-conversion`) | security, external/cwe/cwe-704 | Detects potentially dangerous casts from `char*` to `wchar_t*`.  Results are shown on LGTM by default. |
-| Dead code due to `goto` or `break` statement (`cpp/dead-code-goto`) | maintainability, external/cwe/cwe-561 | Detects dead code following a `goto` or `break` statement. Results are shown on LGTM by default. |
-| Inconsistent direction of for loop (`cpp/inconsistent-loop-direction`) | correctness, external/cwe/cwe-835 | Detects `for` loops where the increment and guard condition don't appear to correspond.  Results are shown on LGTM by default. |
-| Incorrect 'not' operator usage (`cpp/incorrect-not-operator-usage`) | security, external/cwe/cwe-480 | Finds uses of the logical not (`!`) operator that look like they should be bit-wise not (`~`).  Results are hidden on LGTM by default. |
-| Non-virtual destructor in base class (`cpp/virtual-destructor`) | reliability, readability, language-features | This query, `NonVirtualDestructorInBaseClass.ql`, is a replacement in LGTM for the query: No virtual destructor (`AV Rule 78.ql`). The new query ignores base classes with non-public destructors since we consider those to be adequately protected. The new version retains the query identifier, `cpp/virtual-destructor`, and results are displayed by default on LGTM. The old query is no longer run on LGTM. |
-| `NULL` application name with an unquoted path in call to `CreateProcess` (`cpp/unsafe-create-process-call`) | security, external/cwe/cwe-428 | Finds unsafe uses of the `CreateProcess` function.  Results are hidden on LGTM by default. |
-| Setting a DACL to `NULL` in a `SECURITY_DESCRIPTOR` (`cpp/unsafe-dacl-security-descriptor`) | security, external/cwe/cwe-732 | Finds code that creates world-writable objects on Windows by setting their DACL to `NULL`. Results are shown on LGTM by default. |
-
-## Changes to existing LGTM queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Comparison result is always the same (`cpp/constant-comparison`) | Fewer false positive results | Comparisons in template instantiations are now excluded from results. |
-| Empty branch of conditional (`cpp/empty-block`) | Fewer false positive results | Now recognizes commented blocks more reliably. |
-| Expression has no effect (`cpp/useless-expression`) | Fewer false positive results | Expressions in template instantiations are now excluded from results. |
-| Missing return statement (`cpp/missing-return`) | Fewer false positive results, visible by default | Improved results when a function returns a template-dependent type, or makes a non-returning call to another function. Precision increased from 'medium' to 'high' so that alerts are shown by default in LGTM. |
-| Multiplication result converted to larger type (`cpp/integer-multiplication-cast-to-long`) | Fewer false positive results | Char-typed numbers are no longer considered to be potentially large. |
-| No virtual destructor (`cpp/jsf/av-rule-78`) | No results in LGTM | This query is part of the [Joint Strike Fighter](http://www.stroustrup.com/JSF-AV-rules.pdf) suite which defines strict coding rules for air vehicles. Its query identifier has been revised to reflect this. On LGTM this query has been replaced by the similar query "Non-virtual destructor in base class", see New queries above.  The new query highlights only code that is likely to be a problem in the majority of projects. |
-| Overloaded assignment does not return 'this' (`cpp/assignment-does-not-return-this`) | Fewer false positive results | Any return statements that are unreachable are now ignored. |
-| Resource not released in destructor (`cpp/resource-not-released-in-destructor`) | Fewer false positive results | No longer highlights uses of C++ _placement new_ and results are no longer reported for resources where the destructor body is not in the snapshot database. |
-| Self comparison (`cpp/comparison-of-identical-expressions`) | Fewer false positive results | Code inside macro invocations is now excluded from the query. |
-| Static array access may cause overflow (`cpp/static-buffer-overflow`) | More correct results | Data flow to the `size` argument of a buffer operation is now checked in this query. |
-| Suspicious add with sizeof (`cpp/suspicious-add-sizeof`) | Fewer false positive results | Arithmetic with void pointers (where allowed) is now excluded from results. |
-| Unsigned comparison to zero (`cpp/unsigned-comparison-zero`) | Fewer false positive results | Comparisons in template instantiations are now excluded from results. |
-| Wrong type of arguments to formatting function (`cpp/wrong-type-format-argument`) | Fewer false positive results | False positive results involving `typedef`s have been removed.  Expected argument types are determined more accurately, especially for wide string and pointer types.  Custom (non-standard) formatting functions are also identified more accurately. |
-
-## Changes to other queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Array offset used before range check (`cpp/offset-use-before-range-check`) | More results and fewer false positive results | Now recognizes array accesses in different positions within the expression.  Code where the range is checked before and after the array access is no longer highlighted. |
-| AV Rule 164 (`cpp/jsf/av-rule-164`) | Fewer false positive results | Now accounts for explicit casts. |
-| Call to memory access function may overflow buffer (`cpp/overflow-buffer`) | More correct results | Array indexing with a negative index is now detected by this query. |
-| Global could be static (`cpp/jpl-c/limited-scope-file` and `cpp/power-of-10/global-could-be-static`)| Fewer false positive results | Variables with declarations in header files are now excluded from results. |
-| Memory is never freed (`cpp/memory-never-freed`)| Fewer false positive results | No longer highlights uses of C++ _placement new_, which returns a pointer that does not need to be freed. |
-| Negation of unsigned value (`cpp/jsf/av-rule-165`) | Fewer false positive results | Now accounts for explicit casts. |
-| Suspicious call to memset (`cpp/suspicious-call-to-memset`) | Fewer false positive results | Types involving `decltype` are now correctly compared. |
-| Variable scope too large (`cpp/jpl-c/limited-scope-function` and `cpp/power-of-10/variable-scope-too-large`) | Fewer false positive results | Variables with declarations in header files, or that are used at file scope, are now excluded from results. |
-
-## Changes to QL libraries
-
-* New hash consing library (`semmle.code.cpp.valuenumbering.HashCons`) for structural comparison of expressions. Unlike the existing library for global value numbering, this library implements a pure syntactic comparison of expressions and will equate expressions even if they may not compute the same value.
-* The `Buffer.qll` library has more conservative treatment of arrays embedded in structs. This reduces false positive results in a number of security queries, especially `cpp/overflow-buffer`.
-    * Pre-C99 encodings of _flexible array members_ are recognized more reliably.
-    * Arrays of zero size are now treated as a special case.
-* The library `semmle.code.cpp.dataflow.RecursionPrevention` is now deprecated. It was an aid for transitioning data-flow queries from 1.16 to 1.17, and it no longer has any function. Imports of this library should simply be deleted.

change-notes/1.19/analysis-csharp.md

@@ -1,44 +0,0 @@
-# Improvements to C# analysis
-
-## General improvements
-
-### Changes to the autobuilder
-
-During code extraction, when determining the target of `msbuild` or `dotnet build`, the autobuilder now looks for:
-
-* `.proj` files,
-* then `.sln` files,
-* and finally `.csproj`/`.vcxproj` files.
-
-In all three cases, when multiple files of the same type are found, the project/solution file closest to the root is used to build the project.
-
-### Control flow graph improvements
-
-* The control flow graph construction now takes simple Boolean conditions on local scope variables into account. For example, in `if (b) x = 0; if (b) x = 1;`, the control flow graph will reflect the fact that taking the `true` (resp. `false`) branch in the first condition implies taking the same branch in the second condition. In effect, the first assignment to `x` will now be identified as being dead.
-* Code that is only reachable from a constant failing assertion, such as `Debug.Assert(false)`, is considered to be unreachable.
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Uncontrolled format string (`cs/uncontrolled-format-string`) | security, external/cwe/cwe-134 | Finds data flow from remote inputs to the format string in `String.Format`. Results are shown on LGTM by default. |
-| Using a package with a known vulnerability (`cs/use-of-vulnerable-package`) | security, external/cwe/cwe-937 | Finds project build files that import packages with known vulnerabilities. Results are shown on LGTM by default. |
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Cross-site scripting (`cs/web/xss`) | More results | Finds cross-site scripting vulnerabilities in ASP.NET Core applications. |
-| Inconsistent lock sequence (`cs/inconsistent-lock-sequence`) | More results | Finds inconsistent lock sequences globally across calls. |
-| Local scope variable shadows member (`cs/local-shadows-member`) | Fewer results | Results have been removed where a constructor parameter shadows a member, because the parameter is probably used to initialize the member. |
-
-## Changes to code extraction
-
-* Arguments passed using `in` are now extracted.
-* Fixed a bug where the `dynamic` type name was not extracted correctly in certain circumstances.
-* Fixed a bug where method type signatures were extracted incorrectly in some circumstances.
-
-## Changes to QL libraries
-
-* `getArgument()` on `AccessorCall` has been improved so it now takes tuple assignments into account. For example, the argument for the implicit `value` parameter in the setter of property `P` is `0` in `(P, x) = (0, 1)`. Additionally, the argument for the `value` parameter in compound assignments is now only the expanded value, for example, in `P += 7` the argument is `P + 7` and not `7`.
-* The predicate `isInArgument()` has been added to the `AssignableAccess` class. This holds for expressions that are passed as arguments using `in`

change-notes/1.19/analysis-java.md

@@ -1,39 +0,0 @@
-# Improvements to Java analysis
-
-## General improvements
-
-Path explanations have been added to the relevant security queries. 
-Use [QL for Eclipse](https://help.semmle.com/ql-for-eclipse/Content/WebHelp/getting-started.html) 
-to run queries and explore the data flow in results.
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Arbitrary file write during archive extraction ("Zip Slip") (`java/zipslip`) | security, external/cwe/cwe-022 | Identifies extraction routines that allow arbitrary file overwrite vulnerabilities. Results are shown on LGTM by default. |
-| Missing catch of NumberFormatException (`java/uncaught-number-format-exception`) | reliability, external/cwe/cwe-248 | Finds calls to `Integer.parseInt` and similar string-to-number conversions that might raise a `NumberFormatException` without a corresponding `catch`-clause. Results are hidden on LGTM by default. |
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Array index out of bounds (`java/index-out-of-bounds`) | Fewer false positive results | Results for arrays with a length evenly divisible by 3, or some greater number, and an index being increased with a similar stride length are no longer reported. |
-| Confusing overloading of methods (`java/confusing-method-signature`) | Fewer false positive results | A correction to the inheritance relation ensures that spurious results on certain generic classes no longer occur. |
-| Query built from user-controlled sources (`java/sql-injection`) | More results | SQL injection sinks from the Spring JDBC, MyBatis, and Hibernate frameworks are now reported. |
-| Query built without neutralizing special characters (`java/concatenated-sql-query`) | More results | SQL injection sinks from the Spring JDBC, MyBatis, and Hibernate frameworks are now reported. |
-| Unreachable catch clause (`java/unreachable-catch-clause`) | Fewer false positive results | Now accounts for calls to generic methods that throw generic exceptions. |
-| Useless comparison test (`java/constant-comparison`) | Fewer false positive results | Constant comparisons guarding `java.util.ConcurrentModificationException` are no longer reported, as they are intended to always be false in the absence of API misuse. |
-
-## Changes to QL libraries
-
-* The class `ControlFlowNode` (and by extension `BasicBlock`) has until now
-  been directly equatable to `Expr` and `Stmt`.  Exploiting these equalities,
-  for example by using casts, is now deprecated, and the conversions
-  `Expr.getControlFlowNode()` and `Stmt.getControlFlowNode()` should be used
-  instead.
-* The default set of taint sources in the `FlowSources` library is extended to
-  cover parameters annotated with Spring framework annotations indicating
-  remote user input from servlets. This affects all security queries, which
-  will yield additional results on projects that use the Spring Web framework.
-* The `ParityAnalysis` library is replaced with the more general `ModulusAnalysis` library, which improves the range analysis.
-

change-notes/1.19/analysis-javascript.md

@@ -1,82 +0,0 @@
-# Improvements to JavaScript analysis
-
-## General improvements
-
-* Modeling of taint flow through array and buffer operations has been improved. This may give additional results for the security queries.
-
-* Support for AMD modules has been improved. This may give additional results for the security queries, as well as any queries that use type inference on code bases that use such modules.
-
-* Support for popular libraries has been improved. Consequently, queries may produce more results on code bases that use the following features:
-  - File system access, for example, through [fs-extra](https://github.com/jprichardson/node-fs-extra) or [globby](https://www.npmjs.com/package/globby)
-  - Outbound network access, for example, through the [fetch API](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API)
-  - The [lodash](https://lodash.com), [underscore](https://underscorejs.org/), [async](https://www.npmjs.com/package/async) and [async-es](https://www.npmjs.com/package/async-es) libraries
-
-* The taint tracking library now recognizes additional sanitization patterns. This may give fewer false-positive results for the security queries.
-
-* Type inference for function calls has been improved. This may give additional results for queries that rely on type inference.
-
-* Path explanations have been added to the relevant security queries. 
-Use [QL for Eclipse](https://help.semmle.com/ql-for-eclipse/Content/WebHelp/getting-started.html) 
-to run queries and explore the data flow in results.
-
-## New LGTM queries
-
-| **Query**                                     | **Tags**                                             | **Purpose**                                                                                                                                                                 |
-|-----------------------------------------------|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Hard-coded data interpreted as code (`js/hardcoded-data-interpreted-as-code`) | security, external/cwe/cwe-506 | Highlights locations where hard-coded data is transformed and then executed as code or interpreted as an import path, which may indicate embedded malicious code ([CWE-506](https://cwe.mitre.org/data/definitions/506.html)). Results are hidden on LGTM by default. |
-| Host header poisoning in email generation (`js/host-header-forgery-in-email-generation`)|  security, external/cwe/cwe-640 | Highlights code that generates emails with links that can be hijacked by HTTP host header poisoning, indicating a potential violation of [CWE-640](https://cwe.mitre.org/data/definitions/640.html). Results shown on LGTM by default.  |
-| Replacement of a substring with itself (`js/identity-replacement`) | correctness, security, external/cwe/cwe-116 | Highlights string replacements that replace a string with itself, which usually indicates a mistake. Results shown on LGTM by default. |
-| Stored cross-site scripting (`js/stored-xss`) | security, external/cwe/cwe-079, external/cwe/cwe-116 | Highlights uncontrolled stored values flowing into HTML content, indicating a potential violation of [CWE-079](https://cwe.mitre.org/data/definitions/79.html). Results shown on LGTM by default. |
-| Unclear precedence of nested operators (`js/unclear-operator-precedence`) | maintainability, correctness, external/cwe/cwe-783 | Highlights nested binary operators whose relative precedence is easy to misunderstand. Results shown on LGTM by default. |
-| Unneeded defensive code (`js/unneeded-defensive-code`) | correctness, external/cwe/cwe-570, external/cwe/cwe-571 | Highlights locations where defensive code is not needed. Results are shown on LGTM by default. |
-| Unsafe dynamic method access (`js/unsafe-dynamic-method-access` ) | security, external/cwe/cwe-094 | Highlights code that invokes a user-controlled method on an object with unsafe methods. Results are shown on LGTM by default. |
-| Unvalidated dynamic method access (`js/unvalidated-dynamic-method-call` ) | security, external/cwe/cwe-754 | Highlights code that invokes a user-controlled method without guarding against exceptional circumstances. Results are shown on LGTM by default. |
-| Useless assignment to property (`js/useless-assignment-to-property`) | maintainability | Highlights property assignments whose value is always overwritten. Results are shown on LGTM by default. |
-
-## Other new queries
-
-| **Query**                                     | **Tags**                                             | **Purpose**                                                                                                                                                                 |
-|-----------------------------------------------|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Enabling Node.js integration for Electron web content renderers (`js/enabling-electron-renderer-node-integration`) | security, frameworks/electron, external/cwe/cwe-094  | Highlights Electron web content renderer preferences with Node.js integration enabled, indicating a potential violation of [CWE-94](https://cwe.mitre.org/data/definitions/94.html). |
-| File data in outbound network request (`js/file-access-to-http`) | security, external/cwe/cwe-200 | Highlights locations where file data is sent in a network request, indicating a potential violation of [CWE-200](https://cwe.mitre.org/data/definitions/200.html). |
-| User-controlled data written to file (`js/http-to-file-access`) | security, external/cwe/cwe-434, external/cwe/cwe-912 | Highlights locations where user-controlled data is written to a file, indicating a potential violation of [CWE-912](https://cwe.mitre.org/data/definitions/912.html). |
-
-
-## Changes to existing queries
-
-| **Query**                      | **Expected impact**        | **Change**                                   |
-|--------------------------------|----------------------------|----------------------------------------------|
-| Ambiguous HTML id attribute (`js/duplicate-html-id`) | Lower severity | Severity revised to "warning". |
-| Clear-text logging of sensitive information (`js/clear-text-logging`) | Fewer results | Query now tracks flow more precisely. |
-| Client side cross-site scripting (`js/xss`) | More results | HTML injection in the body of an email is also highlighted. |
-| Client-side URL redirect (`js/client-side-unvalidated-url-redirection`) | Fewer false positive results | Safe redirects recognized in more cases. |
-| Conflicting HTML element attributes (`js/conflicting-html-attribute`) | Lower severity | Severity revised to "warning". |
-| Duplicate 'if' condition (`js/duplicate-condition`) | Lower severity | Severity revised to "warning". |
-| Duplicate switch case (`js/duplicate-switch-case`) | Lower severity | Severity revised to "warning". |
-| Inconsistent use of 'new' (`js/inconsistent-use-of-new`) | Simpler result presentation | Results show one call with `new` and one without. |
-| Information exposure through a stack trace (`js/stack-trace-exposure`) | More results | Cases where the entire exception object (including the stack trace) may be exposed are highlighted. |
-| Missing 'this' qualifier (`js/missing-this-qualifier`) | Fewer false positive results | Additional intentional calls to global functions are recognized. |
-| Missing CSRF middleware (`js/missing-token-validation`) | Fewer false positive results | Additional types of CSRF protection middleware are recognized. |
-| Missing variable declaration (`js/missing-variable-declaration`) | Lower severity | Severity revised to "warning". |
-| Regular expression injection (`js/regex-injection`) | Fewer false positive results | Calls to `String.prototype.search` are identified with more precision. |
-| Remote property injection (`js/remote-property-injection`) | Fewer results | No longer highlights dynamic method calls, which are now handled by two new queries: `js/unsafe-dynamic-method-access` and `js/unvalidated-dynamic-method-call`. The precision of this rule has been revised to "medium", reflecting the precision of the remaining results. Results are now hidden on LGTM by default. |
-| Self assignment (`js/redundant-assignment`) | Fewer false positive results | Self-assignments preceded by a JSDoc comment with a `@type` tag are no longer highlighted. |
-| Server-side URL redirect (`js/server-side-unvalidated-url-redirection`) | More results and fewer false positive results | More redirection calls are identified. More safe redirections are recognized and ignored. |
-| Unbound event handler receiver (`js/unbound-event-handler-receiver`) | Fewer false positive results | Additional ways that class methods can be bound are recognized. |
-| Uncontrolled data used in network request (`js/request-forgery`) | More results | Additional kinds of requests are identified. |
-| Unknown directive (`js/unknown-directive`) | Fewer false positives results | YUI compressor directives are now recognized. |
-| Unused variable, import, function or class (`js/unused-local-variable`) | Fewer false positive results and fewer results |  Imports used by the `transform-react-jsx` Babel plugin and fewer variables that may be used by `eval` calls are highlighted. Only one result is reported for an import statement with multiple unused imports. |
-| Useless assignment to local variable (`js/useless-assignment-to-local`) | Fewer false positive results | Additional ways default values can be set are recognized. |
-| Useless conditional (`js/trivial-conditional`) | More results, fewer false positive results | More types of conditional are recognized. Additional defensive coding patterns are now ignored. |
-| Whitespace contradicts operator precedence (`js/whitespace-contradicts-precedence`) | Fewer false positive results | Operators with asymmetric whitespace are no longer highlighted. |
-| Wrong use of 'this' for static method (`js/mixed-static-instance-this-access`) | More results, fewer false-positive results | Inherited methods are now identified. |
-
-## Changes to QL libraries
-
-* A `DataFlow::ParameterNode` instance now exists for all function parameters. Previously, unused parameters did not have a corresponding data-flow node.
-
-* `ReactComponent::getAThisAccess` has been renamed to `getAThisNode`. The old name is still usable but is deprecated. It no longer gets individual `this` expressions, but the `ThisNode` mentioned below.
-
-* The `DataFlow::ThisNode` class now corresponds to the implicit receiver parameter of a function, as opposed to an individual `this` expression. This means that `getALocalSource` now maps all `this` expressions within a given function to the same source. The data-flow node associated with a `ThisExpr` can no longer be cast to `DataFlow::SourceNode` or `DataFlow::ThisNode`. Using `getALocalSource` before casting, or instead of casting, is recommended.
-
-* The flow configuration framework now supports distinguishing and tracking different kinds of taint, specified by an extensible class `FlowLabel` (which can also be referred to by its alias `TaintKind`).

change-notes/1.19/analysis-python.md

@@ -1,94 +0,0 @@
-# Improvements to Python analysis
-
-## General improvements
-
-### Representation of the control flow graph
-
-The representation of the control flow graph (CFG) has been modified to better reflect the semantics of Python. As part of these changes, a new predicate `Stmt.getAnEntryNode()` has been added to make it easier to write reachability queries involving statements.
-
-#### CFG nodes removed
-
-The following statement types no longer have a CFG node for the statement itself, as their sub-expressions already contain all the
-semantically significant information:
-
-* `ExprStmt`
-* `If`
-* `Assign`
-* `Import`
-
-For example, the CFG for `if cond: foo else bar` now starts with the CFG node for `cond`.
-
-#### CFG nodes reordered
-
-For the following statement types, the CFG node for the statement now follows the CFG nodes of its sub-expressions to follow Python semantics:
-
-* `Print`
-* `TemplateWrite`
-* `ImportStar`
-
-For example the CFG for `print foo` (in Python 2) has changed from `print -> foo` to `foo -> print`, to reflect the runtime behavior.
-
-The CFG for the `with` statement has been re-ordered to more closely reflect the semantics.
-For the `with` statement:
-```python
-with cm as var:
-    body
-```
-
-* Previous CFG node order: `<with>` -> `cm` -> `var` -> `body`
-* New CFG node order: `cm` -> `<with>` -> `var` -> `body`
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Assert statement tests the truth value of a literal constant (`py/assert-literal-constant`) | reliability, correctness     | Checks whether an assert statement is testing the truth of a literal constant value. Results are hidden on LGTM by default. |
-| Flask app is run in debug mode (`py/flask-debug`) | security, external/cwe/cwe-215, external/cwe/cwe-489 | Finds instances where a Flask application is run in debug mode. Results are shown on LGTM by default. |
-| Information exposure through an exception (`py/stack-trace-exposure`) | security, external/cwe/cwe-209, external/cwe/cwe-497 | Finds instances where information about an exception may be leaked to an external user. Results are shown on LGTM by default. |
-| Jinja2 templating with autoescape=False (`py/jinja2/autoescape-false`) | security, external/cwe/cwe-079 | Finds instantiations of `jinja2.Environment` with `autoescape=False` which may allow XSS attacks. Results are hidden on LGTM by default. |
-| Request without certificate validation (`py/request-without-cert-validation`) | security, external/cwe/cwe-295 | Finds requests where certificate verification has been explicitly turned off, possibly allowing man-in-the-middle attacks. Results are hidden on LGTM by default. |
-| Use of weak cryptographic key (`py/weak-crypto-key`) | security, external/cwe/cwe-326 | Finds creation of weak cryptographic keys. Results are shown on LGTM by default. |
-
-## Changes to existing queries
-
-All taint-tracking queries now support visualization of paths in QL for Eclipse.
-Most security alerts are now visible on LGTM by default. This means that you may see results that were previously hidden for the following queries: 
-
-* Code injection (`py/code-injection`)
-* Reflected server-side cross-site scripting (`py/reflective-xss`)
-* SQL query built from user-controlled sources (`py/sql-injection`)
-* Uncontrolled data used in path expression (`py/path-injection`)
-* Uncontrolled command line (`py/command-line-injection`)
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Command injection (`py/command-line-injection`) | More results | Additional sinks in the `os`, and `popen` modules may find more results in some projects. |
-| Encoding error (`py/encoding-error`) | Better alert location | Alerts are now shown at the start of the encoding error, rather than at the top of the file. |
-| Missing call to \_\_init\_\_ during object initialization (`py/missing-call-to-init`) | Fewer false positive results | Results where it is likely that the full call chain has not been analyzed are no longer reported. |
-| URL redirection from remote source (`py/url-redirection`) | Fewer false positive results | Taint is no longer tracked from the right-hand side of binary expressions. In other words `SAFE + TAINTED` is now treated as safe. |
-
-
-## Changes to code extraction
-
-## Improved reporting of encoding errors
-
-The extractor now outputs the location of the first character that triggers an `EncodingError`. 
-Any queries that report encoding errors will now show results at the location of the character that caused the error.
-
-### Improved scalability
-
-Scaling is near linear to at least 20 CPU cores.
-
-### Improved logging
-
-* Five levels of logging are available: `CRITICAL`, `ERROR`, `WARN`, `INFO` and `DEBUG`. `WARN` is the default.
-* LGTM uses `INFO` level logging. QL tools use `WARN` level logging by default.
-* The `--verbose` flag can be specified specified multiple times to increase the logging level once per flag added.
-* The `--quiet` flag can be specified multiple times to reduce the logging level once per flag added.
-* Log lines are now in the `[SEVERITY] message` style and never overlap.
-
-## Changes to QL libraries
-
-* Taint-tracking analysis now understands HTTP requests in the `twisted` library.
-* The analysis now handles `isinstance` and `issubclass` tests involving the basic abstract base classes better. For example, the test `issubclass(list, collections.Sequence)` is now understood to be `True`
-* Taint tracking automatically tracks tainted mappings and collections, without you having to add additional taint kinds. This means that custom taints are tracked from `x` to `y` in the following flow: `l = [x]; y =l[0]`.

change-notes/1.19/extractor-javascript.md

@@ -1,27 +0,0 @@
-[[ condition: enterprise-only ]]
-
-# Improvements to JavaScript analysis
-
-## General improvements
-
-* On LGTM, files whose name ends in `.min.js` or `-min.js` are no longer extracted by default. These files usually contain minified code and any alerts in these files would be hidden by default. If you still want to extract code from these files, you can add the following filters to your `lgtm.yml` file (or add them to existing filters):
-
-```yaml
-extraction:
-  javascript:
-    index:
-      filters:
-        - include: "**/*.min.js"
-        - include: "**/*-min.js"
-```
-
-* The TypeScript compiler is now included in the LGTM Enterprise and QL command-line tools installations, and you no longer need to install it manually.
-  If you need to override the compiler version, set the `SEMMLE_TYPESCRIPT_HOME` environment variable to
-  point to an installation of the `typescript` NPM package.
-
-## Changes to code extraction
-
-The extractor now supports:
-
-* [Optional Chaining](https://github.com/tc39/proposal-optional-chaining) expressions.
-* Additional [Flow](https://flow.org/) syntax.

change-notes/1.19/support/framework-support.rst

@@ -1,61 +0,0 @@
-Frameworks and libraries
-########################
-
-The QL libraries and queries in this version have been explicitly checked against the libraries and frameworks listed below.
-
-.. pull-quote::
-
-    Tip
-    
-    If you're interested in other libraries or frameworks, you can extend the analysis to cover them. 
-    For example, by extending the data flow libraries to include data sources and sinks for additional libraries or frameworks.
-
-.. There is currently no built-in support for libraries or frameworks for C/C++.
-
-C# built-in support
-================================
-
-* ASP.Net MVC framework
-* ASP.NET Web API
-* ASP.NET Web Forms
-* ASP.NET Core
-* ASP.NET Core MVC
-* ASP.Net Core Razor
-* Razor templates
-
-
-COBOL built-in support
-===================================
-
-* Embedded SQL
-* Embedded CICS
-
-
-Java built-in support
-==================================
-
-.. csv-table:: 
-     :file: java-frameworks.csv
-     :header-rows: 1
-     :class: fullWidthTable
-     :widths: auto
-
-
-JavaScript and TypeScript built-in support
-=======================================================
-
-.. csv-table:: 
-     :file: javascript-typescript-frameworks.csv
-     :header-rows: 1
-     :class: fullWidthTable
-     :widths: auto
-
-
-Python built-in support
-====================================
-
-.. csv-table:: 
-     :file: python-frameworks.csv
-     :header-rows: 1
-     :class: fullWidthTable
-     :widths: auto

change-notes/1.19/support/java-frameworks.csv

@@ -1,10 +0,0 @@
-Name, Category
-Hibernate, Database
-iBatis / MyBatis, Database
-Java Persistence API (JPA), Database
-JDBC, Database
-Kryo deserialization, Serialization
-SnakeYaml, Serialization
-Spring JDBC, Database
-Spring MVC, Web application framework
-XStream, Serialization

change-notes/1.19/support/javascript-typescript-frameworks.csv

@@ -1,22 +0,0 @@
-Name, Category
-angularjs, HTML framework
-axios, Network communicator
-browser, Runtime environment
-electron, Runtime environment
-express, Server
-hapi, Server
-jquery, Utility library
-koa, Server
-lodash, Utility library
-mongodb, Database
-mssql, Database
-mysql, Database
-node, Runtime environment
-postgres, Database
-ramda, Utility library
-react, HTML framework
-request, Network communicator
-sequelize, Database
-sqlite3, Database
-superagent, Network communicator
-underscore, Utility library

change-notes/1.19/support/language-support.rst

@@ -1,19 +0,0 @@
-Languages and compilers
-#######################
-
-QL and LGTM version |version| support analysis of the following languages compiled by the following compilers.
-
-Note that where there are several versions or dialects of a language, the supported variants are listed.
-
-.. csv-table::
-     :file: versions-compilers.csv
-     :header-rows: 1
-     :widths: auto
-     :stub-columns: 1
-
-.. container:: footnote-group
-
-    .. [1] The best results are achieved with COBOL code that stays close to the ANSI 85 standard.  
-    .. [2] Java 11 refers to the language features used. Builds that execute on Java 6 or higher can be analyzed.
-    .. [3] JSX and Flow code, YAML, JSON, and HTML files may also be analyzed with JavaScript files. 
-    .. [4] TypeScript analysis is performed by running the JavaScript extractor with TypeScript enabled. This is the default for LGTM.   

change-notes/1.19/support/python-frameworks.csv

@@ -1,7 +0,0 @@
-Name, Category
-Django, Web application framework
-Flask, Microframework
-Pyramid, Web application framework
-Tornado, Web application framework and asynchronous networking library
-Twisted, Networking engine
-WebOb, WSGI request library