API graphs: Prune nodes before computing trackUseNode

.bazelrc

@@ -1,3 +0,0 @@
-build --repo_env=CC=clang --repo_env=CXX=clang++ --copt="-std=c++17"
-
-try-import %workspace%/local.bazelrc

.bazelversion

@@ -1 +0,0 @@
-5.0.0

.codeqlmanifest.json

@@ -1,30 +1,10 @@
 {
     "provide": [
-        "*/ql/src/qlpack.yml",
-        "*/ql/lib/qlpack.yml",
-        "*/ql/test/qlpack.yml",
-        "*/ql/examples/qlpack.yml",
-        "*/ql/consistency-queries/qlpack.yml",
-        "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
-        "go/ql/config/legacy-support/qlpack.yml",
-        "go/build/codeql-extractor-go/codeql-extractor.yml",
-        "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
-        "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml",
-        "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
-        "csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
-        "csharp/ql/campaigns/Solorigate/src/qlpack.yml",
-        "csharp/ql/campaigns/Solorigate/test/qlpack.yml",
-        "misc/legacy-support/*/qlpack.yml",
-        "misc/suite-helpers/qlpack.yml",
-        "ruby/extractor-pack/codeql-extractor.yml",
-        "swift/extractor-pack/codeql-extractor.yml",
-        "ql/extractor-pack/codeql-extractor.yml"
-    ],
-    "versionPolicies": {
-      "default": {
-        "requireChangeNotes": true,
-        "committedPrereleaseSuffix": "dev",
-        "committedVersion": "nextPatchRelease"
-      }
-    }
+        "ql/lib/qlpack.yml",
+        "ql/src/qlpack.yml",
+        "ql/consistency-queries/qlpack.yml",
+        "ql/test/qlpack.yml",
+        "ql/examples/qlpack.yml",
+        "extractor-pack/codeql-extractor.yml"
+    ]
 }

.devcontainer/Dockerfile

@@ -0,0 +1,15 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.162.0/containers/rust/.devcontainer/base.Dockerfile
+
+FROM mcr.microsoft.com/vscode/devcontainers/rust:0-1
+
+RUN apt-key --keyring /usr/share/keyrings/githubcli-archive-keyring.gpg adv \
+    --keyserver keyserver.ubuntu.com --recv-key C99B11DEB97541F0 && \
+    echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages $(lsb_release -cs) main" \
+    |  tee /etc/apt/sources.list.d/github-cli2.list > /dev/null
+
+
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install --no-install-recommends gh 
+
+COPY post_create.sh /bin/post_create.sh
+COPY post_attach.sh /bin/post_attach.sh

.devcontainer/devcontainer.json

@@ -1,16 +1,39 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.162.0/containers/rust
 {
-  "extensions": [
-    "rust-lang.rust",
-    "bungcip.better-toml",
-    "github.vscode-codeql",
-    "hbenl.vscode-test-explorer",
-    "ms-vscode.test-adapter-converter",
-    "slevesque.vscode-zipexplorer"
-  ],
-  "settings": {
-    "files.watcherExclude": {
-      "**/target/**": true
-    },
-    "codeQL.runningQueries.memory": 2048
-  }
-}
+	"name": "Rust",
+	"build": {
+		"dockerfile": "Dockerfile"
+	},
+	"runArgs": [
+		"--cap-add=SYS_PTRACE",
+		"--security-opt",
+		"seccomp=unconfined"
+	],
+	// Set *default* container specific settings.json values on container create.
+	"settings": {
+		"terminal.integrated.shell.linux": "/bin/bash",
+		"lldb.executable": "/usr/bin/lldb",
+		// VS Code don't watch files under ./target
+		"files.watcherExclude": {
+			"**/target/**": true
+		}
+	},
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"rust-lang.rust",
+		"bungcip.better-toml",
+		"vadimcn.vscode-lldb",
+		"mutantdino.resourcemonitor",
+		"ms-azuretools.vscode-docker",
+		"github.vscode-codeql"
+	],
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "rustc --version",
+	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode",
+	"postCreateCommand": [ "/bin/post_create.sh" ],
+	"postAttachCommand": [ "flock", "-E", "0", "-n", "/var/lock/post_attach.lock", "/bin/post_attach.sh" ]
+}

.devcontainer/post_attach.sh

@@ -0,0 +1,37 @@
+#! /bin/bash
+set -xe
+
+echo "Check installed CodeQL version"
+CURRENT_CODEQL_BIN=$(readlink -e /usr/local/bin/codeql || echo "")
+LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
+
+BASE_DIR=/home/vscode/codeql-binaries
+mkdir -p "${BASE_DIR}"
+LATEST_CODEQL_DIR="${BASE_DIR}/codeql-${LATEST}"
+LATEST_CODEQL_BIN="${LATEST_CODEQL_DIR}/codeql/codeql"
+
+if [ "${CURRENT_CODEQL_BIN}" != "${LATEST_CODEQL_BIN}" ]; then
+  echo "Installing CodeQL ${LATEST}"
+  TMPDIR=$(mktemp -d -p "$(dirname ${LATEST_CODEQL_DIR})")
+  gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip -D "${TMPDIR}" "$LATEST" 
+  unzip -oq "${TMPDIR}/codeql-linux64.zip" -d "${TMPDIR}"
+  rm -f "${TMPDIR}/codeql-linux64.zip"
+  mv "${TMPDIR}" "${LATEST_CODEQL_DIR}"
+  test -x "${LATEST_CODEQL_BIN}" && sudo ln -sf "${LATEST_CODEQL_BIN}" /usr/local/bin/codeql
+  if [[ "${CURRENT_CODEQL_BIN}" =~ .*/codeql/codeql ]]; then
+    rm -rf "$(dirname $(dirname ${CURRENT_CODEQL_BIN}))"
+  fi
+fi
+
+echo "Build the Ruby extractor"
+
+# clone the git dependencies using "git clone" because cargo's builtin git support is rather slow
+REPO_DIR="${CARGO_HOME:-/home/vscode/.cargo}/git/db" 
+REPO_DIR_ERB="${REPO_DIR}/tree-sitter-embedded-template-4c796e3340c233b6"
+REPO_DIR_RUBY="${REPO_DIR}/tree-sitter-ruby-666a40ce046f8e7a"
+
+mkdir -p "${REPO_DIR}"
+test -e "${REPO_DIR_ERB}" || git clone -q --bare https://github.com/tree-sitter/tree-sitter-embedded-template "${REPO_DIR_ERB}"
+test -e "${REPO_DIR_RUBY}" || git clone -q --bare https://github.com/tree-sitter/tree-sitter-ruby.git "${REPO_DIR_RUBY}"
+
+scripts/create-extractor-pack.sh

.devcontainer/post_create.sh

@@ -0,0 +1,4 @@
+#! /bin/bash
+
+mkdir -p /home/vscode/.config/codeql
+echo '--search-path /workspaces/codeql-ruby' >> /home/vscode/.config/codeql/config

.editorconfig

@@ -1,2 +0,0 @@
-[*]
-end_of_line = lf

.gitattributes

@@ -1,73 +1 @@
-# Text files will be normalized to LF line endings in the Git database, and will keep those LF line
-# endings in the working tree even on Windows. If you make changes below, you should renormalize the
-# affected files by running the following from the root of this repo (requires Git 2.16 or greater):
-#
-# git add --renormalize .
-# git status  [just to show what files were renormalized]
-# git commit -m "Normalize line endings"
-
-# Anything Git auto-detects as text gets normalized and checked out as LF
-* text=auto eol=lf
-
-# Explicitly set a bunch of known extensions to text, in case auto detection gets confused.
-*.ql text
-*.qll text
-*.qlref text
-*.dbscheme text
-*.qhelp text
-*.html text
-*.htm text
-*.xhtml text
-*.xhtm text
-*.js text
-*.mjs text
-*.ts text
-*.json text
-*.yml text
-*.yaml text
-*.c text
-*.cpp text
-*.h text
-*.hpp text
-*.md text
-*.stats text
-*.xml text
-*.sh text
-*.pl text
-*.java text
-*.cs text
-*.py text
-*.lua text
-*.expected text
-*.go text
-
-# Explicitly set a bunch of known extensions to binary, because Git < 2.10 will treat
-# `* text=auto eol=lf` as `* text eol=lf`
-*.png -text
-*.jpg -text
-*.jpeg -text
-*.gif -text
-*.dll -text
-*.pdb -text
-
-java/ql/test/stubs/**/*.java linguist-generated=true
-java/ql/test/experimental/stubs/**/*.java linguist-generated=true
-
-# Force git not to modify line endings for go or html files under the go/ql directory
-go/ql/**/*.go -text
-go/ql/**/*.html -text
-# Force git not to modify line endings for go dbschemes
-go/*.dbscheme -text
-# Preserve unusual line ending from codeql-go merge
-go/extractor/opencsv/CSVReader.java -text
-
-# For some languages, upgrade script testing references really old dbscheme
-# files from legacy upgrades that have CRLF line endings. Since upgrade
-# resolution relies on object hashes, we must suppress line ending conversion
-# for those testing dbscheme files.
-*/ql/lib/upgrades/initial/*.dbscheme -text
-
-# Generated test files - these are synced from the standard JavaScript libraries using
-# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge
+Cargo.lock   -diff -whitespace

.github/ISSUE_TEMPLATE/lgtm-com---false-positive.md

@@ -1,24 +0,0 @@
----
-name: LGTM.com - false positive
-about: Tell us about an alert that shouldn't be reported
-title: LGTM.com - false positive
-labels: false-positive
-assignees: ''
-
----
-
-**Description of the false positive**
-
-<!-- Please explain briefly why you think it shouldn't be included. -->
-
-**URL to the alert on the project page on LGTM.com**
-
-<!--
-1. Open the project on LGTM.com.
-For example, https://lgtm.com/projects/g/pallets/click/.
-2. Switch to the `Alerts` tab. For example, https://lgtm.com/projects/g/pallets/click/alerts/.
-3. Scroll to the alert that you would like to report.
-4. Click on the right most icon `View this alert within the complete file`.
-5. A new browser tab opens. Copy and paste the page URL here.
-For example, https://lgtm.com/projects/g/pallets/click/snapshot/719fb7d8322b0767cdd1e5903ba3eb3233ba8dd5/files/click/_winconsole.py#xa08d213ab3289f87:1.
--->

.github/ISSUE_TEMPLATE/ql---general.md

@@ -1,14 +0,0 @@
----
-name: General issue
-about: Tell us if you think something is wrong or if you have a question
-title: General issue
-labels: question
-assignees: ''
-
----
-
-**Description of the issue**
-
-<!-- Please explain briefly what is the problem.
-If it is about an LGTM project, please include its URL.-->
-

.github/actions/create-extractor-pack/action.yml

@@ -0,0 +1,15 @@
+name: Build Ruby CodeQL pack
+description: Builds the Ruby CodeQL pack
+runs:
+  using: composite
+  steps:
+    - uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          target
+        key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
+    - name: Build Extractor
+      shell: bash
+      run: env "PATH=$PATH:${{ github.workspace }}/codeql" scripts/create-extractor-pack.sh

.github/actions/fetch-codeql/action.yml

@@ -3,22 +3,11 @@ description: Fetches the latest version of CodeQL
 runs:
   using: composite
   steps:
-    - name: Select platform - Linux
-      if: runner.os == 'Linux'
-      shell: bash
-      run: echo "GA_CODEQL_CLI_PLATFORM=linux64" >> $GITHUB_ENV
-
-    - name: Select platform - MacOS
-      if: runner.os == 'MacOS'
-      shell: bash
-      run: echo "GA_CODEQL_CLI_PLATFORM=osx64" >> $GITHUB_ENV
-
     - name: Fetch CodeQL
       shell: bash
       run: |
         LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
-        gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-$GA_CODEQL_CLI_PLATFORM.zip "$LATEST"
-        unzip -q -d "${RUNNER_TEMP}" codeql-$GA_CODEQL_CLI_PLATFORM.zip
-        echo "${RUNNER_TEMP}/codeql" >> "${GITHUB_PATH}"
+        gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
+        unzip -q codeql-linux64.zip
       env:
         GITHUB_TOKEN: ${{ github.token }}

.github/codeql/codeql-config.yml

@@ -1,11 +0,0 @@
-name: "CodeQL config"
-
-queries:
-  - uses: security-and-quality
-
-paths-ignore:
-  - '/cpp/'
-  - '/java/'
-  - '/python/'
-  - '/javascript/ql/test'
-  - '/javascript/extractor/tests'

.github/dependabot.yml

@@ -1,26 +1,18 @@
 version: 2
 updates:
   - package-ecosystem: "cargo"
-    directory: "ruby/node-types"
+    directory: "/node-types"
     schedule:
       interval: "daily"
   - package-ecosystem: "cargo"
-    directory: "ruby/generator"
+    directory: "/generator"
     schedule:
       interval: "daily"
   - package-ecosystem: "cargo"
-    directory: "ruby/extractor"
+    directory: "/extractor"
     schedule:
       interval: "daily"
   - package-ecosystem: "cargo"
-    directory: "ruby/autobuilder"
+    directory: "/autobuilder"
     schedule:
       interval: "daily"
-
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "daily"
-    ignore:
-      - dependency-name: '*'
-        update-types: ['version-update:semver-patch', 'version-update:semver-minor']

.github/labeler.yml

@@ -1,35 +0,0 @@
-"C++":
-  - cpp/**/*
-  - change-notes/**/*cpp*
-
-"C#":
-  - csharp/**/*
-  - change-notes/**/*csharp*
-
-Java:
-  - java/**/*
-  - change-notes/**/*java.*
-
-JS:
-  - any: [ 'javascript/**/*', '!javascript/ql/experimental/adaptivethreatmodeling/**/*' ]
-  - change-notes/**/*javascript*
-
-Python:
-  - python/**/*
-  - change-notes/**/*python*
-
-Ruby:
-  - ruby/**/*
-  - change-notes/**/*ruby*
-  
-Swift:
-  - swift/**/*
-  - change-notes/**/*swift*
-
-documentation:
-  - "**/*.qhelp"
-  - "**/*.md"
-  - docs/**/*
-
-"QL-for-QL": 
-  - ql/**/*

.github/problem-matchers/codeql-query-format.json

@@ -1,14 +0,0 @@
-{
-    "problemMatcher": [
-        {
-            "owner": "codeql-query-format",
-            "pattern": [
-                {
-                    "regexp": "^((.*) would change by autoformatting\\.)$",
-                    "file": 2,
-                    "message": 1
-                }
-            ]
-        }
-    ]
-}

.github/problem-matchers/codeql-syntax-check.json

@@ -1,17 +0,0 @@
-{
-    "problemMatcher": [
-        {
-            "owner": "codeql-syntax-check",
-            "pattern": [
-                {
-                    "regexp": "^((ERROR|WARNING): .* \\((.*):(\\d+),(\\d+)-\\d+\\))$",
-                    "message": 1,
-                    "file": 3,
-                    "line": 4,
-                    "col": 5,
-                    "severity": 2
-                }
-            ]
-        }
-    ]
-}

.github/problem-matchers/codeql-test-run.json

@@ -1,14 +0,0 @@
-{
-    "problemMatcher": [
-        {
-            "owner": "codeql-test-run",
-            "pattern": [
-                {
-                    "regexp": "(\\[.*\\] FAILED\\((RESULT|COMPILATION)\\) (.*))$",
-                    "file": 3,
-                    "message": 1
-                }
-            ]
-        }
-    ]
-}

.github/problem-matchers/make.json

@@ -1,13 +0,0 @@
-{
-    "problemMatcher": [
-        {
-            "owner": "make",
-            "pattern": [
-                {
-                    "regexp": "^(make: \\*\\*\\* .*)$",
-                    "message": 1
-                }
-            ]
-        }
-    ]
-}

.github/workflows/build.yml

@@ -0,0 +1,229 @@
+name: Build / Release
+
+on:
+  push:
+    branches:
+      - main
+      - 'rc/*'
+  pull_request:
+    branches:
+      - main
+      - 'rc/*'
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Version tag to create"
+        required: false
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install GNU tar
+        if: runner.os == 'macOS'
+        run: |
+          brew install gnu-tar
+          echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('**/Cargo.lock') }}
+      - name: Check formatting
+        run: cargo fmt --all -- --check
+      - name: Build
+        run: cargo build --verbose
+      - name: Run tests
+        run: cargo test --verbose
+      - name: Release build
+        run: cargo build --release
+      - name: Generate dbscheme
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: target/release/ruby-generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+      - uses: actions/upload-artifact@v2
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        with:
+          name: ruby.dbscheme
+          path: ql/lib/ruby.dbscheme
+      - uses: actions/upload-artifact@v2
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        with:
+          name: TreeSitter.qll
+          path: ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+      - uses: actions/upload-artifact@v2
+        with:
+          name: extractor-${{ matrix.os }}
+          path: |
+            target/release/ruby-autobuilder
+            target/release/ruby-autobuilder.exe
+            target/release/ruby-extractor
+            target/release/ruby-extractor.exe
+          retention-days: 1
+  compile-queries:
+    runs-on: ubuntu-latest
+    env:
+      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
+    steps:
+      - uses: actions/checkout@v2
+      - name: Fetch CodeQL
+        run: |
+          LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
+          gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
+          unzip -q codeql-linux64.zip
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - name: Build Query Pack
+        run: |
+          codeql/codeql pack create ql/lib --output target/packs
+          codeql/codeql pack install ql/src
+          codeql/codeql pack create ql/src --output target/packs
+          PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
+          codeql/codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
+          (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
+      - name: Compile with previous CodeQL versions
+        run: |
+          for version in  $(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | sort --version-sort | tail -3 | head -2); do
+            rm -f codeql-linux64.zip
+            gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$version"
+            rm -rf codeql; unzip -q codeql-linux64.zip
+            codeql/codeql query compile target/packs/*
+          done
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: codeql-ruby-queries
+          path: |
+            target/packs/*
+          retention-days: 1
+
+  package:
+    runs-on: ubuntu-latest
+    needs: [build, compile-queries]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+        with:
+          name: ruby.dbscheme
+          path: ruby
+      - uses: actions/download-artifact@v2
+        with:
+          name: extractor-ubuntu-latest
+          path: linux64
+      - uses: actions/download-artifact@v2
+        with:
+          name: extractor-windows-latest
+          path: win64
+      - uses: actions/download-artifact@v2
+        with:
+          name: extractor-macos-latest
+          path: osx64
+      - run: |
+          mkdir -p ruby
+          cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
+          mkdir -p ruby/tools/{linux64,osx64,win64}
+          cp linux64/ruby-autobuilder ruby/tools/linux64/autobuilder
+          cp osx64/ruby-autobuilder ruby/tools/osx64/autobuilder
+          cp win64/ruby-autobuilder.exe ruby/tools/win64/autobuilder.exe
+          cp linux64/ruby-extractor ruby/tools/linux64/extractor
+          cp osx64/ruby-extractor ruby/tools/osx64/extractor
+          cp win64/ruby-extractor.exe ruby/tools/win64/extractor.exe
+          chmod +x ruby/tools/{linux64,osx64}/{autobuilder,extractor}
+          zip -rq codeql-ruby.zip ruby
+      - uses: actions/upload-artifact@v2
+        with:
+          name: codeql-ruby-pack
+          path: codeql-ruby.zip
+          retention-days: 1
+      - uses: actions/download-artifact@v2
+        with:
+          name: codeql-ruby-queries
+          path: qlpacks
+      - run: |
+          echo '{
+            "provide": [
+            "ruby/codeql-extractor.yml",
+            "qlpacks/*/*/*/qlpack.yml"
+            ]
+          }' > .codeqlmanifest.json
+          zip -rq codeql-ruby-bundle.zip .codeqlmanifest.json ruby qlpacks
+      - uses: actions/upload-artifact@v2
+        with:
+          name: codeql-ruby-bundle
+          path: codeql-ruby-bundle.zip
+          retention-days: 1
+      - if: ${{ github.event.inputs.tag }}
+        run: |
+          gh release create --prerelease \
+             --title "CodeQL Ruby (${{ github.event.inputs.tag }})" \
+             --target "${{ github.sha }}" \
+             "${{ github.event.inputs.tag }}" \
+             codeql-ruby-bundle.zip
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    runs-on: ${{ matrix.os }}
+    needs: [package]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          repository: Shopify/example-ruby-app
+          ref: 67a0decc5eb550f3a9228eda53925c3afd40dfe9
+      - name: Fetch CodeQL
+        shell: bash
+        run: |
+          LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
+          gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql.zip "$LATEST"
+          unzip -q codeql.zip
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        working-directory: ${{ runner.temp }}
+      - name: Download Ruby bundle
+        uses: actions/download-artifact@v2
+        with:
+          name: codeql-ruby-bundle
+          path: ${{ runner.temp }}
+      - name: Unzip Ruby bundle
+        shell: bash
+        run: unzip -q -d "${{ runner.temp }}/ruby-bundle" "${{ runner.temp }}/codeql-ruby-bundle.zip"
+      - name: Prepare test files
+        shell: bash
+        run: |
+          echo "import ruby select count(File f)" > "test.ql"
+          echo "| 4 |" > "test.expected"
+          echo 'name: sample-tests
+          version: 0.0.0
+          dependencies:
+            codeql/ruby-all: 0.0.1
+          extractor: ruby
+          tests: .
+          ' > qlpack.yml
+      - name: Run QL test
+        shell: bash
+        run: |
+          "${{ runner.temp }}/codeql/codeql" test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
+      - name: Create database
+        shell: bash
+        run: |
+          "${{ runner.temp }}/codeql/codeql" database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
+      - name: Analyze database
+        shell: bash
+        run: |
+          "${{ runner.temp }}/codeql/codeql" database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/check-change-note.yml

@@ -1,27 +0,0 @@
-name: Check change note
-
-on:
-  pull_request_target:
-    types: [labeled, unlabeled, opened, synchronize, reopened, ready_for_review]
-    paths:
-      - "*/ql/src/**/*.ql"
-      - "*/ql/src/**/*.qll"
-      - "*/ql/lib/**/*.ql"
-      - "*/ql/lib/**/*.qll"
-      - "!**/experimental/**"
-      - "!ql/**"
-      - ".github/workflows/check-change-note.yml"
-
-jobs:
-  check-change-note:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Fail if no change note found. To fix, either add one, or add the `no-change-note-required` label.
-        if: |
-          github.event.pull_request.draft == false &&
-          !contains(github.event.pull_request.labels.*.name, 'no-change-note-required')
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' |
-          grep true -c

.github/workflows/check-qldoc.yml

@@ -1,54 +0,0 @@
-name: "Check QLdoc coverage"
-
-on:
-  pull_request:
-    paths:
-      - "*/ql/lib/**"
-      - .github/workflows/check-qldoc.yml
-    branches:
-      - main
-      - "rc/*"
-
-jobs:
-  qldoc:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Install CodeQL
-        run: |
-          gh extension install github/gh-codeql
-          gh codeql set-channel nightly
-          gh codeql version
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-
-      - name: Check QLdoc coverage
-        shell: bash
-        run: |
-          EXIT_CODE=0
-          # TODO: remove the swift exception from the regex when we fix generated QLdoc
-          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!swift)[a-z]*/ql/lib' || true; } | sort -u)"
-          for pack_dir in ${changed_lib_packs}; do
-            lang="${pack_dir%/ql/lib}"
-            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
-          done
-          git checkout HEAD^
-          for pack_dir in ${changed_lib_packs}; do
-            # When we add a new language, pack_dir would not exist in HEAD^.
-            # In this case the right thing to do is to skip the check.
-            [[ ! -d "${pack_dir}" ]] && continue
-            lang="${pack_dir%/ql/lib}"
-            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
-            awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-current.txt" | sort -u > "${RUNNER_TEMP}/current-undocumented.txt"
-            awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-baseline.txt" | sort -u > "${RUNNER_TEMP}/baseline-undocumented.txt"
-            UNDOCUMENTED="$(grep -f <(comm -13 "${RUNNER_TEMP}/baseline-undocumented.txt" "${RUNNER_TEMP}/current-undocumented.txt") "${RUNNER_TEMP}/${lang}-current.txt" || true)"
-            if [ -n "$UNDOCUMENTED" ]; then
-              echo "$UNDOCUMENTED" | awk -F, '{gsub(/"/,""); print "::warning file='"${pack_dir}"'/"$1",line="$2"::Missing QLdoc for "$5, $3 }'
-              EXIT_CODE=1
-            fi
-          done
-          exit "${EXIT_CODE}"

.github/workflows/close-stale.yml

@@ -1,30 +0,0 @@
-name: Mark stale issues
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "30 1 * * *"
-
-jobs:
-  stale:
-    if: github.repository == 'github/codeql'
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/stale@v5
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Comment or remove the `Stale` label in order to avoid having this issue closed in 7 days.'
-        close-issue-message: 'This issue was closed because it has been inactive for 7 days.'
-        days-before-stale: 14
-        days-before-close: 7
-        only-labels: awaiting-response
-
-        # do not mark PRs as stale
-        days-before-pr-stale: -1
-        days-before-pr-close: -1
-
-        # Uncomment for dry-run
-        # debug-only: true
-        # operations-per-run: 1000

.github/workflows/codeql-analysis.yml

@@ -1,62 +0,0 @@
-name: "Code scanning - action"
-
-on:
-  push:
-    branches:
-      - main
-      - 'rc/*'
-  pull_request:
-    branches:
-      - main
-      - 'rc/*'
-    paths:
-      - 'csharp/**'
-      - '.github/codeql/**'
-      - '.github/workflows/codeql-analysis.yml'
-  schedule:
-    - cron: '0 9 * * 1'
-
-jobs:
-  CodeQL-Build:
-
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: read
-      security-events: write
-      pull-requests: read
-
-    steps:
-    - name: Setup dotnet
-      uses: actions/setup-dotnet@v2
-      with:
-        dotnet-version: 6.0.202
-
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@main
-      # Override language selection by uncommenting this and choosing your languages
-      with:
-        languages: csharp
-        config-file: ./.github/codeql/codeql-config.yml
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    #- name: Autobuild
-    #  uses: github/codeql-action/autobuild@main
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    - run: |
-       dotnet build csharp /p:UseSharedCompilation=false
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@main

.github/workflows/csv-coverage-metrics.yml

@@ -1,70 +0,0 @@
-name: "Publish framework coverage as metrics"
-
-on:
-  schedule:
-    - cron: '5 0 * * *'
-  push:
-    branches:
-      - main
-  workflow_dispatch:
-  pull_request:
-    branches:
-      - main
-    paths:
-      - ".github/workflows/csv-coverage-metrics.yml"
-
-jobs:
-  publish-java:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Setup CodeQL
-        uses: ./.github/actions/fetch-codeql
-      - name: Create empty database
-        run: |
-          DATABASE="${{ runner.temp }}/java-database"
-          PROJECT="${{ runner.temp }}/java-project"
-          mkdir -p "$PROJECT/src/tmp/empty"
-          echo "class Empty {}" >> "$PROJECT/src/tmp/empty/Empty.java"
-          codeql database create "$DATABASE" --language=java --source-root="$PROJECT" --command 'javac src/tmp/empty/Empty.java'
-      - name: Capture coverage information
-        run: |
-          DATABASE="${{ runner.temp }}/java-database"
-          codeql database analyze --format=sarif-latest --output=metrics-java.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
-      - uses: actions/upload-artifact@v3
-        with:
-          name: metrics-java.sarif
-          path: metrics-java.sarif
-          retention-days: 20
-      - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@main
-        with:
-          sarif_file: metrics-java.sarif
-  
-  publish-csharp:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Setup CodeQL
-        uses: ./.github/actions/fetch-codeql
-      - name: Create empty database
-        run: |
-          DATABASE="${{ runner.temp }}/csharp-database"
-          PROJECT="${{ runner.temp }}/csharp-project"
-          dotnet new classlib --language=C# --output="$PROJECT"
-          codeql database create "$DATABASE" --language=csharp --source-root="$PROJECT" --command 'dotnet build /t:rebuild csharp-project.csproj /p:UseSharedCompilation=false'
-      - name: Capture coverage information
-        run: |
-          DATABASE="${{ runner.temp }}/csharp-database"
-          codeql database analyze --format=sarif-latest --output=metrics-csharp.sarif -- "$DATABASE" ./csharp/ql/src/Metrics/Summaries/FrameworkCoverage.ql
-      - uses: actions/upload-artifact@v3
-        with:
-          name: metrics-csharp.sarif
-          path: metrics-csharp.sarif
-          retention-days: 20
-      - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@main
-        with:
-          sarif_file: metrics-csharp.sarif

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -1,99 +0,0 @@
-name: Check framework coverage changes
-
-on:
-  pull_request:
-    paths:
-      - '.github/workflows/csv-coverage-pr-comment.yml'
-      - '*/ql/src/**/*.ql'
-      - '*/ql/src/**/*.qll'
-      - '*/ql/lib/**/*.ql'
-      - '*/ql/lib/**/*.qll'
-      - 'misc/scripts/library-coverage/*.py'
-      # input data files
-      - '*/documentation/library-coverage/cwe-sink.csv'
-      - '*/documentation/library-coverage/frameworks.csv'
-    branches:
-      - main
-      - 'rc/*'
-
-jobs:
-  generate:
-    name: Generate framework coverage artifacts
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Dump GitHub context
-      env:
-        GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-      run: echo "$GITHUB_CONTEXT"
-    - name: Clone self (github/codeql) - MERGE
-      uses: actions/checkout@v3
-      with:
-        path: merge
-    - name: Clone self (github/codeql) - BASE
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 2
-        path: base
-    - run: |
-        git checkout HEAD^1
-        git log -1 --format='%H'
-      working-directory: base
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Generate CSV files on merge commit of the PR
-      run: |
-        echo "Running generator on merge"
-        PATH="$PATH:codeql-cli/codeql" python merge/misc/scripts/library-coverage/generate-report.py ci merge merge
-        mkdir out_merge
-        cp framework-coverage-*.csv out_merge/
-        cp framework-coverage-*.rst out_merge/
-    - name: Generate CSV files on base commit of the PR
-      run: |
-        echo "Running generator on base"
-        PATH="$PATH:codeql-cli/codeql" python base/misc/scripts/library-coverage/generate-report.py ci base base
-        mkdir out_base
-        cp framework-coverage-*.csv out_base/
-        cp framework-coverage-*.rst out_base/
-    - name: Generate diff of coverage reports
-      run: |
-        python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
-    - name: Upload CSV package list
-      uses: actions/upload-artifact@v3
-      with:
-        name: csv-framework-coverage-merge
-        path: |
-          out_merge/framework-coverage-*.csv
-          out_merge/framework-coverage-*.rst
-    - name: Upload CSV package list
-      uses: actions/upload-artifact@v3
-      with:
-        name: csv-framework-coverage-base
-        path: |
-          out_base/framework-coverage-*.csv
-          out_base/framework-coverage-*.rst
-    - name: Upload comparison results
-      uses: actions/upload-artifact@v3
-      with:
-        name: comparison
-        path: |
-          comparison.md
-    - name: Save PR number
-      run: |
-        mkdir -p pr
-        echo ${{ github.event.pull_request.number }} > pr/NR
-    - name: Upload PR number
-      uses: actions/upload-artifact@v3
-      with:
-        name: pr
-        path: pr/

.github/workflows/csv-coverage-pr-comment.yml

@@ -1,34 +0,0 @@
-name: Comment on PR with framework coverage changes
-
-on:
-  workflow_run:
-    workflows: ["Check framework coverage changes"]
-    types:
-      - completed
-
-jobs:
-  check:
-    name: Check framework coverage differences and comment
-    runs-on: ubuntu-latest
-    if: >
-      ${{ github.event.workflow_run.event == 'pull_request' &&
-      github.event.workflow_run.conclusion == 'success' }}
-
-    steps:
-    - name: Dump GitHub context
-      env:
-        GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-      run: echo "$GITHUB_CONTEXT"
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.8
-
-    - name: Check coverage difference file and comment
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        RUN_ID: ${{ github.event.workflow_run.id }}
-      run: |
-        python misc/scripts/library-coverage/comment-pr.py "$GITHUB_REPOSITORY" "$RUN_ID"

.github/workflows/csv-coverage-timeseries.yml

@@ -1,42 +0,0 @@
-name: Build framework coverage timeseries reports
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
-      with:
-        path: script
-    - name: Clone self (github/codeql) for analysis
-      uses: actions/checkout@v3
-      with:
-        path: codeqlModels
-        fetch-depth: 0
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Build modeled package list
-      run: |
-        CLI=$(realpath "codeql-cli/codeql")
-        echo $CLI
-        PATH="$PATH:$CLI" python script/misc/scripts/library-coverage/generate-timeseries.py codeqlModels
-    - name: Upload timeseries CSV
-      uses: actions/upload-artifact@v3
-      with:
-        name: framework-coverage-timeseries
-        path: framework-coverage-timeseries-*.csv
-

.github/workflows/csv-coverage-update.yml

@@ -1,44 +0,0 @@
-name: Update framework coverage reports
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "0 0 * * *"
-
-jobs:
-  update:
-    name: Update framework coverage report
-    if: github.repository == 'github/codeql'
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Dump GitHub context
-      env:
-        GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-      run: echo "$GITHUB_CONTEXT"
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
-      with:
-        path: ql
-        fetch-depth: 0
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-
-    - name: Generate coverage files
-      run: |
-        PATH="$PATH:codeql-cli/codeql" python ql/misc/scripts/library-coverage/generate-report.py ci ql ql
-
-    - name: Create pull request with changes
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-        python ql/misc/scripts/library-coverage/create-pr.py ql "$GITHUB_REPOSITORY"

.github/workflows/csv-coverage.yml

@@ -1,49 +0,0 @@
-name: Build framework coverage reports
-
-on:
-  workflow_dispatch:
-    inputs:
-      qlModelShaOverride:
-        description: 'github/codeql repo SHA used for looking up the CSV models'
-        required: false
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
-      with:
-        path: script
-    - name: Clone self (github/codeql) for analysis
-      uses: actions/checkout@v3
-      with:
-        path: codeqlModels
-        ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Build modeled package list
-      run: |
-        PATH="$PATH:codeql-cli/codeql" python script/misc/scripts/library-coverage/generate-report.py ci codeqlModels script
-    - name: Upload CSV package list
-      uses: actions/upload-artifact@v3
-      with:
-        name: framework-coverage-csv
-        path: framework-coverage-*.csv
-    - name: Upload RST package list
-      uses: actions/upload-artifact@v3
-      with:
-        name: framework-coverage-rst
-        path: framework-coverage-*.rst
-

.github/workflows/dataset_measure.yml

@@ -0,0 +1,71 @@
+name: Collect database stats
+
+on:
+  push:
+    branches:
+      - main
+      - 'rc/*'
+    paths:
+      - ql/lib/ruby.dbscheme
+  pull_request:
+    branches:
+      - main
+      - 'rc/*'
+    paths:
+      - ql/lib/ruby.dbscheme
+  workflow_dispatch:
+
+jobs:
+  measure:
+    env:
+      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
+    strategy:
+      fail-fast: false
+      matrix:
+        repo: [rails/rails, discourse/discourse, spree/spree]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - uses: ./.github/actions/create-extractor-pack
+
+      - name: Checkout ${{ matrix.repo }}
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ matrix.repo }}
+          path: ${{ github.workspace }}/repo
+      - name: Create database
+        run: |
+          codeql/codeql database create \
+            --search-path "${{ github.workspace }}" \
+            --threads 4 \
+            --language ruby --source-root "${{ github.workspace }}/repo" \
+            "${{ runner.temp }}/database"
+      - name: Measure database
+        run: |
+          mkdir -p "stats/${{ matrix.repo }}"
+          codeql/codeql dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ruby"
+      - uses: actions/upload-artifact@v2
+        with:
+          name: measurements
+          path: stats
+          retention-days: 1
+
+  merge:
+    runs-on: ubuntu-latest
+    needs: measure
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+        with:
+          name: measurements
+          path: stats
+      - run: |
+          python -m pip install --user lxml
+          find stats -name 'stats.xml' | sort | xargs python scripts/merge_stats.py --output ql/lib/ruby.dbscheme.stats --normalise ruby_tokeninfo
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ruby.dbscheme.stats
+          path: ql/lib/ruby.dbscheme.stats

.github/workflows/go-tests.yml

@@ -1,161 +0,0 @@
-name: "Go: Run Tests"
-on:
-  pull_request:
-    paths:
-      - "go/**"
-      - .github/workflows/go-tests.yml
-jobs:
-
-  test-linux:
-    name: Test Linux (Ubuntu)
-    runs-on: ubuntu-latest
-    steps:
-
-    - name: Set up Go 1.18.1
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.18.1
-      id: go
-
-    - name: Set up CodeQL CLI
-      run: |
-        echo "Removing old CodeQL Directory..."
-        rm -rf $HOME/codeql
-        echo "Done"
-        cd $HOME
-        echo "Downloading CodeQL CLI..."
-        LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | sort --version-sort | grep -v beta | tail -1)
-        gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
-        echo "Done"
-        echo "Unpacking CodeQL CLI..."
-        unzip -q codeql-linux64.zip
-        rm -f codeql-linux64.zip
-        echo "Done"
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    - name: Enable problem matchers in repository
-      shell: bash
-      run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-    - name: Build
-      run: |
-        cd go
-        env PATH=$PATH:$HOME/codeql make
-
-    - name: Check that all QL and Go code is autoformatted
-      run: |
-        cd go
-        env PATH=$PATH:$HOME/codeql make check-formatting
-
-    - name: Compile qhelp files to markdown
-      run: |
-        cd go
-        env PATH=$PATH:$HOME/codeql QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown
-
-    - name: Upload qhelp markdown
-      uses: actions/upload-artifact@v2
-      with:
-        name: qhelp-markdown
-        path: go/qhelp-out/**/*.md
-
-    - name: Test
-      run: |
-        cd go
-        env PATH=$PATH:$HOME/codeql make test
-
-  test-mac:
-    name: Test MacOS
-    runs-on: macOS-latest
-    steps:
-    - name: Set up Go 1.18.1
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.18.1
-      id: go
-
-    - name: Set up CodeQL CLI
-      run: |
-        echo "Removing old CodeQL Directory..."
-        rm -rf $HOME/codeql
-        echo "Done"
-        cd $HOME
-        echo "Downloading CodeQL CLI..."
-        LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | sort --version-sort | grep -v beta | tail -1)
-        gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-osx64.zip "$LATEST"
-        echo "Done"
-        echo "Unpacking CodeQL CLI..."
-        unzip -q codeql-osx64.zip
-        rm -f codeql-osx64.zip
-        echo "Done"
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    - name: Enable problem matchers in repository
-      shell: bash
-      run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-    - name: Build
-      run: |
-        cd go
-        env PATH=$PATH:$HOME/codeql make
-
-    - name: Test
-      run: |
-        cd go
-        env PATH=$PATH:$HOME/codeql make test
-
-  test-win:
-    name: Test Windows
-    runs-on: windows-2019
-    steps:
-    - name: Set up Go 1.18.1
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.18.1
-      id: go
-
-    - name: Set up CodeQL CLI
-      run: |
-        echo "Removing old CodeQL Directory..."
-        rm -rf $HOME/codeql
-        echo "Done"
-        cd "$HOME"
-        echo "Downloading CodeQL CLI..."
-        LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | sort --version-sort | grep -v beta | tail -1)
-        gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-win64.zip "$LATEST"
-        echo "Done"
-        echo "Unpacking CodeQL CLI..."
-        unzip -q -o codeql-win64.zip
-        unzip -q -o codeql-win64.zip codeql/codeql.exe
-        rm -f codeql-win64.zip
-        echo "Done"
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-      shell:
-        bash
-
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    - name: Enable problem matchers in repository
-      shell: bash
-      run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-    - name: Build
-      run: |
-        $Env:Path += ";$HOME\codeql"
-        cd go
-        make
-
-    - name: Test
-      run: |
-        $Env:Path += ";$HOME\codeql"
-        cd go
-        make test

.github/workflows/js-ml-tests.yml

@@ -1,76 +0,0 @@
-name: JS ML-powered queries tests
-
-on:
-  push:
-    paths:
-      - "javascript/ql/experimental/adaptivethreatmodeling/**"
-      - .github/workflows/js-ml-tests.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "javascript/ql/experimental/adaptivethreatmodeling/**"
-      - .github/workflows/js-ml-tests.yml
-
-defaults:
-  run:
-    working-directory: javascript/ql/experimental/adaptivethreatmodeling
-
-jobs:
-  qlformat:
-    name: Check QL formatting
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: ./.github/actions/fetch-codeql
-
-      - name: Check QL formatting
-        run: |
-          find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
-            xargs -0 codeql query format --check-only
-
-  qlcompile:
-    name: Check QL compilation
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: ./.github/actions/fetch-codeql
-
-      - name: Install pack dependencies
-        run: |
-          for pack in modelbuilding src; do
-            codeql pack install --mode verify -- "${pack}"
-          done
-
-      - name: Check QL compilation
-        run: |
-          codeql query compile \
-            --check-only \
-            --ram 5120 \
-            --additional-packs "${{ github.workspace }}" \
-            --threads=0 \
-            -- \
-            lib modelbuilding src
-
-  qltest:
-    name: Run QL tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: ./.github/actions/fetch-codeql
-
-      - name: Install pack dependencies
-        run: codeql pack install -- test
-
-      - name: Run QL tests
-        run: |
-          codeql test run \
-            --threads=0 \
-            --ram 5120 \
-            --additional-packs "${{ github.workspace }}" \
-            -- \
-            test

.github/workflows/labeler.yml

@@ -1,11 +0,0 @@
-name: "Pull Request Labeler"
-on:
-- pull_request_target
-
-jobs:
-  triage:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/labeler@v4
-      with:
-        repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/mad_modelDiff.yml

@@ -1,103 +0,0 @@
-name: Models as Data - Diff
-
-on:
-  workflow_dispatch:
-    inputs:
-      projects:
-        description: "The projects to generate models for"
-        required: true
-        default: '["netty/netty"]'
-  pull_request:
-    branches:
-      - main
-    paths:
-      - "java/ql/src/utils/model-generator/**/*.*"
-      - ".github/workflows/mad_modelDiff.yml"
-
-permissions:
-  contents: read
-
-jobs:
-  model-diff:
-    name: Model Difference
-    runs-on: ubuntu-latest
-    if: github.repository == 'github/codeql'
-    strategy:
-      matrix:
-        slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
-    steps:
-      - name: Clone github/codeql from PR
-        uses: actions/checkout@v3
-        if: github.event.pull_request
-        with:
-          path: codeql-pr
-      - name: Clone github/codeql from main
-        uses: actions/checkout@v3
-        with:
-          path: codeql-main
-          ref: main
-      - uses: ./codeql-main/.github/actions/fetch-codeql
-      - name: Download database
-        env:
-          SLUG: ${{ matrix.slug }}
-        run: |
-          set -x
-          mkdir lib-dbs
-          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
-          projectId=`curl -s https://lgtm.com/api/v1.0/projects/g/${SLUG} | jq .id`
-          curl -L "https://lgtm.com/api/v1.0/snapshots/$projectId/java" -o "$SHORTNAME.zip"
-          unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
-          mkdir "lib-dbs/$SHORTNAME/"
-          mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/"
-      - name: Generate Models (PR and main)
-        run: |
-          set -x
-          mkdir tmp-models
-          MODELS=`pwd`/tmp-models
-          DATABASES=`pwd`/lib-dbs
-
-          analyzeDatabaseWithCheckout() {
-            QL_VARIANT=$1
-            DATABASE=$2
-            cd codeql-$QL_VARIANT
-            SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
-            mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
-            cd ..
-          }
-
-          for d in $DATABASES/*/ ; do
-            ls -1 "$d"
-
-            analyzeDatabaseWithCheckout "main" $d
-            if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]
-            then
-              analyzeDatabaseWithCheckout "pr" $d
-            fi
-          done
-      - name: Install diff2html
-        if: github.event.pull_request
-        run: |
-          npm install -g diff2html-cli
-      - name: Generate Model Diff
-        if: github.event.pull_request
-        run: |
-          set -x
-          MODELS=`pwd`/tmp-models
-          ls -1 tmp-models/
-          for m in $MODELS/*_main.qll ; do
-            t="${m/main/"pr"}"
-            basename=`basename $m`
-            name="diff_${basename/_main.qll/""}"
-            (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
-          done
-      - uses: actions/upload-artifact@v3
-        with:
-          name: models
-          path: tmp-models/*.qll
-          retention-days: 20
-      - uses: actions/upload-artifact@v3
-        with:
-          name: diffs
-          path: tmp-models/*.html
-          retention-days: 20

.github/workflows/mad_regenerate-models.yml

@@ -1,62 +0,0 @@
-name: Regenerate framework models
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "30 2 * * *"
-  pull_request:
-    branches:
-      - main
-    paths:
-      - ".github/workflows/mad_regenerate-models.yml"
-
-jobs:
-  regenerate-models:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        # placeholder required for each axis, excluded below, replaced by the actual combinations (see include)
-        slug: ["placeholder"]
-        ref: ["placeholder"]
-        include:
-          - slug: "apache/commons-io"
-            ref: "8985de8fe74f6622a419b37a6eed0dbc484dc128"
-        exclude:
-          - slug: "placeholder"
-            ref: "placeholder"
-    steps:
-      - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
-      - name: Setup CodeQL binaries
-        uses: ./.github/actions/fetch-codeql
-      - name: Clone repositories
-        uses: actions/checkout@v3
-        with:
-          path: repos/${{ matrix.ref }}
-          ref: ${{ matrix.ref }}
-          repository: ${{ matrix.slug }}
-      - name: Build database
-        env:
-          SLUG: ${{ matrix.slug }}
-          REF: ${{ matrix.ref }}
-        run: |
-          mkdir dbs
-          cd repos/${REF}
-          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
-          codeql database create --language=java ../../dbs/${SHORTNAME}
-      - name: Regenerate models in-place
-        env:
-          SLUG: ${{ matrix.slug }}
-        run: |
-          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
-          java/ql/src/utils/model-generator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
-      - name: Stage changes
-        run: |
-          find java -name "*.qll" -print0 | xargs -0 git add
-          git status
-          git diff --cached > models.patch
-      - uses: actions/upload-artifact@v3
-        with:
-          name: patch
-          path: models.patch
-          retention-days: 7

.github/workflows/post-pr-comment.yml

@@ -1,74 +0,0 @@
-# This workflow is the second part of the process described in
-# .github/workflows/qhelp-pr-preview.yml
-# See that file for more info.
-
-name: Post PR comment
-on:
-  workflow_run:
-    workflows: [Render QHelp changes]
-    types:
-      - completed
-
-permissions:
-  pull-requests: write
-  actions: read
-
-jobs:
-  post_comment:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download artifact
-        run: gh run download "${WORKFLOW_RUN_ID}" --repo "${GITHUB_REPOSITORY}" --name "comment"
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-          WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
-
-      - name: Check that PR SHA matches workflow SHA
-        run: |
-          PR="$(grep -o '^[0-9]\+$' pr_number.txt)"
-          PR_HEAD_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/pulls/${PR}" --jq .head.sha)"
-          # Check that the pull-request head SHA matches the head SHA of the workflow run
-          if [ "${WORKFLOW_RUN_HEAD_SHA}" != "${PR_HEAD_SHA}" ]; then
-            echo "PR head SHA ${PR_HEAD_SHA} does not match workflow_run event SHA ${WORKFLOW_RUN_HEAD_SHA}. Stopping." 1>&2
-            exit 1
-          fi
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-          WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_commit.id }}
-
-      - name: Create or update comment
-        run: |
-          COMMENT_PREFIX="QHelp previews"
-          COMMENT_AUTHOR="github-actions[bot]"
-          PR_NUMBER="$(grep -o '^[0-9]\+$' pr_number.txt)"
-
-          # If there is no existing comment, comment_id.txt will contain just a
-          # newline (due to jq & gh behaviour). This will cause grep to fail, so
-          # we catch that.
-          RAW_COMMENT_ID=$(grep -o '^[0-9]\+$' comment_id.txt || true)
-
-          if [ $RAW_COMMENT_ID ]
-          then
-            # Fetch existing comment, and validate:
-            # - comment belongs to the PR with number $PR_NUMBER
-            # - comment starts with the expected prefix ("QHelp previews")
-            # - comment author is github-actions[bot]
-            FILTER='select(.issue_url | endswith($repo+"/issues/"+$pr))
-                  | select(.body | startswith($prefix))
-                  | select(.user.login == $author)
-                  | .id'
-            COMMENT_ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${RAW_COMMENT_ID}" | jq --arg repo "${GITHUB_REPOSITORY}" --arg pr "${PR_NUMBER}" --arg prefix "${COMMENT_PREFIX}" --arg author "${COMMENT_AUTHOR}" "${FILTER}")
-            if [ $COMMENT_ID ]
-            then
-              # Update existing comment
-              jq --rawfile body comment_body.txt '{"body":$body}' -n | gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${COMMENT_ID}" -X PATCH --input -
-            else
-              echo "Comment ${RAW_COMMENT_ID} did not pass validations: not editing." >&2
-              exit 1
-            fi
-          else
-            # Create new comment
-            jq --rawfile body comment_body.txt '{"body":$body}' -n | gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -X POST --input -
-          fi
-        env:
-          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/qhelp-pr-preview.yml

@@ -1,102 +0,0 @@
-# This workflow checks for any changes in .qhelp files in pull requests.
-# For any changed files, it renders them to markdown in a file called `comment_body.txt`.
-# It then checks if there's an existing comment on the pull request generated by
-# this workflow, and writes the comment ID to `comment_id.txt`.
-# It also writes the PR number to `pr_number.txt`.
-# These three files are uploaded as an artifact.
-
-# When this workflow completes, the workflow "Post PR comment" runs.
-# It downloads the artifact and adds a comment to the PR with the rendered
-# QHelp.
-
-# The task is split like this because creating PR comments requires extra
-# permissions that we don't want to expose to PRs from external forks.
-
-# For more info see:
-# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_run
-# https://docs.github.com/en/actions/security-guides/automatic-token-authentication#permissions-for-the-github_token
-name: Render QHelp changes
-
-permissions:
-  contents: read
-  pull-requests: read
-
-on:
-  pull_request:
-    branches:
-      - main
-      - "rc/*"
-    paths:
-      - "ruby/**/*.qhelp"
-
-jobs:
-  qhelp:
-    runs-on: ubuntu-latest
-    steps:
-      - run: echo "${PR_NUMBER}" > pr_number.txt
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-      - uses: actions/upload-artifact@v3
-        with:
-          name: comment
-          path: pr_number.txt
-          if-no-files-found: error
-          retention-days: 1
-
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-          persist-credentials: false
-      - uses: ./.github/actions/fetch-codeql
-      - name: Determine changed files
-        id: changes
-        run: |
-          (git diff -z --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep -z '.qhelp$' | grep -z -v '.inc.qhelp';
-           git diff -z --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep -z '.inc.qhelp$' | xargs --null -rn1 basename | xargs --null -rn1 git grep -z -l) |
-           grep -z '.qhelp$' | grep -z -v '^-' | sort -z -u > "${RUNNER_TEMP}/paths.txt"
-
-      - name: QHelp preview
-        run: |
-          EXIT_CODE=0
-          echo "QHelp previews:" > comment_body.txt
-          while read -r -d $'\0' path; do
-            if [ ! -f "${path}" ]; then
-               exit 1
-            fi
-            echo "<details> <summary>${path}</summary>"
-            echo
-            codeql generate query-help --format=markdown -- "./${path}" 2> errors.txt || EXIT_CODE="$?"
-            if [ -s errors.txt ]; then
-               echo "# errors/warnings:"
-               echo '```'
-               cat errors.txt
-               cat errors.txt 1>&2
-               echo '```'
-            fi
-            echo "</details>"
-          done < "${RUNNER_TEMP}/paths.txt" >> comment_body.txt
-          exit "${EXIT_CODE}"
-
-      - if: always()
-        uses: actions/upload-artifact@v3
-        with:
-          name: comment
-          path: comment_body.txt
-          if-no-files-found: error
-          retention-days: 1
-
-      - name: Save ID of existing QHelp comment (if it exists)
-        run: |
-          # Find the latest comment starting with "QHelp previews"
-          COMMENT_PREFIX="QHelp previews"
-          gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate | jq --arg prefix "${COMMENT_PREFIX}" '[.[] | select(.body|startswith($prefix)) | .id] | max' > comment_id.txt
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-          PR_NUMBER: ${{ github.event.number }}
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: comment
-          path: comment_id.txt
-          if-no-files-found: error
-          retention-days: 1

.github/workflows/qhelp.yml

@@ -0,0 +1,39 @@
+name: Query help preview
+
+on:
+  pull_request:
+    branches:
+      - main
+      - 'rc/*'
+    paths:
+      - "**/*.qhelp"
+
+jobs:
+  qhelp:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Determine changed files
+        id: changes
+        run: |
+          echo -n "::set-output name=qhelp_files::"
+          (git diff --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep .qhelp$ | grep -v .inc.qhelp;
+           git diff --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep .inc.qhelp$ | xargs -d '\n' -rn1 basename | xargs -d '\n' -rn1 git grep -l) |
+           sort -u | xargs -d '\n' -n1 printf "'%s' "
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: QHelp preview
+        if: ${{ steps.changes.outputs.qhelp_files }}
+        run: |
+          ( echo "QHelp previews:";
+          for path in ${{ steps.changes.outputs.qhelp_files }} ; do
+            echo "<details> <summary>${path}</summary>"
+            echo
+            codeql/codeql generate query-help --format=markdown ${path}
+            echo "</details>"
+          done) | gh pr comment "${{ github.event.pull_request.number }}" -F -
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/ql-for-ql-build.yml

@@ -1,203 +0,0 @@
-name: Run QL for QL
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  queries:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
-        with:
-          languages: javascript # does not matter
-          tools: latest
-      - name: Get CodeQL version
-        id: get-codeql-version
-        run: |
-          echo "::set-output name=version::$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)"
-        shell: bash
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Cache queries
-        id: cache-queries
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/query-pack.zip
-          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
-      - name: Build query pack
-        if: steps.cache-queries.outputs.cache-hit != 'true'
-        run: |
-          cd ql/ql/src
-          "${CODEQL}" pack create
-          cd .codeql/pack/codeql/ql/0.0.0
-          zip "${PACKZIP}" -r .
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-          PACKZIP: ${{ runner.temp }}/query-pack.zip
-      - name: Upload query pack
-        uses: actions/upload-artifact@v3
-        with:
-          name: query-pack-zip
-          path: ${{ runner.temp }}/query-pack.zip
-
-  extractors:
-    strategy:
-      fail-fast: false
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: Cache entire extractor
-        id: cache-extractor
-        uses: actions/cache@v3
-        with:
-          path: |
-            ql/target/release/ql-autobuilder
-            ql/target/release/ql-autobuilder.exe
-            ql/target/release/ql-extractor
-            ql/target/release/ql-extractor.exe
-          key: ${{ runner.os }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
-      - name: Cache cargo
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
-      - name: Check formatting
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; cargo fmt --all -- --check
-      - name: Build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; cargo build --verbose
-      - name: Run tests
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; cargo test --verbose
-      - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; cargo build --release
-      - name: Generate dbscheme
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v3
-        with:
-          name: extractor-ubuntu-latest
-          path: |
-            ql/target/release/ql-autobuilder
-            ql/target/release/ql-autobuilder.exe
-            ql/target/release/ql-extractor
-            ql/target/release/ql-extractor.exe
-          retention-days: 1
-  package:
-    runs-on: ubuntu-latest
-
-    needs:
-      - extractors
-      - queries
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
-        with:
-          name: query-pack-zip
-          path: query-pack-zip
-      - uses: actions/download-artifact@v3
-        with:
-          name: extractor-ubuntu-latest
-          path: linux64
-      - run: |
-          unzip query-pack-zip/*.zip -d pack
-          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats pack/
-          mkdir -p pack/tools/linux64
-          if [[ -f linux64/ql-autobuilder ]]; then
-            cp linux64/ql-autobuilder pack/tools/linux64/autobuilder
-            chmod +x pack/tools/linux64/autobuilder
-          fi
-          if [[ -f linux64/ql-extractor ]]; then
-            cp linux64/ql-extractor pack/tools/linux64/extractor
-            chmod +x pack/tools/linux64/extractor
-          fi
-          cd pack
-          zip -rq ../codeql-ql.zip .
-      - uses: actions/upload-artifact@v3
-        with:
-          name: codeql-ql-pack
-          path: codeql-ql.zip
-          retention-days: 1
-  analyze:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        folder: [cpp, csharp, java, javascript, python, ql, ruby, swift, go]
-
-    needs:
-      - package
-
-    steps:
-      - name: Download pack
-        uses: actions/download-artifact@v3
-        with:
-          name: codeql-ql-pack
-          path: ${{ runner.temp }}/codeql-ql-pack-artifact
-
-      - name: Prepare pack
-        run: |
-          unzip "${PACK_ARTIFACT}/*.zip" -d "${PACK}"
-        env:
-          PACK_ARTIFACT: ${{ runner.temp }}/codeql-ql-pack-artifact
-          PACK: ${{ runner.temp }}/pack
-      - name: Hack codeql-action options
-        run: |
-          JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
-          echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
-        env:
-          PACK: ${{ runner.temp }}/pack
-
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Create CodeQL config file
-        run: |
-          echo "paths:" > ${CONF}
-          echo "  - ${FOLDER}" >> ${CONF}
-          echo "paths-ignore:" >> ${CONF}
-          echo "  - ql/ql/test" >> ${CONF} 
-          echo "disable-default-queries: true" >> ${CONF}
-          echo "packs:" >> ${CONF}
-          echo "  - codeql/ql" >> ${CONF}
-          echo "Config file: "
-          cat ${CONF}
-        env: 
-          CONF: ./ql-for-ql-config.yml
-          FOLDER: ${{ matrix.folder }}
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
-        with:
-          languages: ql
-          db-location: ${{ runner.temp }}/db
-          config-file: ./ql-for-ql-config.yml
-          tools: latest
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@aa93aea877e5fb8841bcb1193f672abf6e9f2980
-        with: 
-          category: "ql-for-ql-${{ matrix.folder }}"
-      - name: Copy sarif file to CWD
-        run: cp ../results/ql.sarif ./${{ matrix.folder }}.sarif
-      - name: Sarif as artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.folder }}.sarif
-          path: ${{ matrix.folder }}.sarif
-

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -1,83 +0,0 @@
-name: Collect database stats for QL for QL
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - ql/ql/src/ql.dbscheme
-  pull_request:
-    branches: [main]
-    paths:
-      - ql/ql/src/ql.dbscheme
-  workflow_dispatch:
-
-jobs:
-  measure:
-    env:
-      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
-    strategy:
-      matrix:
-        repo:
-          - github/codeql
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
-        with:
-          languages: javascript # does not matter
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
-      - name: Build Extractor
-        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v3
-        with:
-          repository: ${{ matrix.repo }}
-          path: ${{ github.workspace }}/repo
-      - name: Create database
-        run: |
-          "${CODEQL}" database create \
-            --search-path "ql/extractor-pack" \
-            --threads 4 \
-            --language ql --source-root "${{ github.workspace }}/repo" \
-            "${{ runner.temp }}/database"
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Measure database
-        run: |
-          mkdir -p "stats/${{ matrix.repo }}"
-          "${CODEQL}" dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ql"
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - uses: actions/upload-artifact@v3
-        with:
-          name: measurements
-          path: stats
-          retention-days: 1
-
-  merge:
-    runs-on: ubuntu-latest
-    needs: measure
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
-        with:
-          name: measurements
-          path: stats
-      - run: |
-          python -m pip install --user lxml
-          find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ruby/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ql.dbscheme.stats
-          path: ql/ql/src/ql.dbscheme.stats

.github/workflows/ql-for-ql-tests.yml

@@ -1,52 +0,0 @@
-name: Run QL for QL Tests
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - "ql/**"
-  pull_request:
-    branches: [main]
-    paths:
-      - "ql/**"
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  qltest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
-        with:
-          languages: javascript # does not matter
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ql/target
-          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
-      - name: Build extractor
-        run: |
-          cd ql;
-          codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
-          env "PATH=$PATH:$codeqlpath" ./create-extractor-pack.sh
-      - name: Run QL tests
-        run: |
-          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Check QL formatting
-        run: |
-          find ql/ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Check QL compilation
-        run: |
-          "${CODEQL}" query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}/ql/extractor-pack" "ql/ql/src" "ql/ql/examples"
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}

.github/workflows/qltest.yml

@@ -0,0 +1,42 @@
+name: Run QL Tests
+
+on:
+  push:
+    branches:
+      - main
+      - 'rc/*'
+  pull_request:
+    branches:
+      - main
+      - 'rc/*'
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  qltest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/fetch-codeql
+      - uses: ./.github/actions/create-extractor-pack
+      - name: Run QL tests
+        run: |
+          codeql/codeql pack install ql/test
+          codeql/codeql test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --additional-packs "${HOME}/.codeql/packages/codeql/suite-helpers/0.0.1" --consistency-queries ql/consistency-queries ql/test
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - name: Check QL formatting
+        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql/codeql query format --check-only
+      - name: Check QL compilation
+        run: |
+          codeql/codeql pack install ql/src
+          codeql/codeql query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}" "ql/src" "ql/examples"
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - name: Check DB upgrade scripts
+        run: |
+          echo >empty.trap
+          codeql/codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
+          codeql/codeql dataset upgrade testdb --additional-packs ql/lib/upgrades
+          diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme

.github/workflows/query-list.yml

@@ -1,39 +0,0 @@
-name: Build code scanning query list
-
-on:
-  push:
-    branches:
-     - main
-     - 'rc/**'
-  pull_request:
-    paths:
-      - '.github/workflows/query-list.yml'
-      - 'misc/scripts/generate-code-scanning-query-list.py'
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
-      with:
-        path: codeql 
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      # Look under the `codeql` directory, as this is where we checked out the `github/codeql` repo
-      uses: ./codeql/.github/actions/fetch-codeql
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Build code scanning query list
-      run: |
-        python codeql/misc/scripts/generate-code-scanning-query-list.py > code-scanning-query-list.csv
-    - name: Upload code scanning query list
-      uses: actions/upload-artifact@v3
-      with:
-        name: code-scanning-query-list
-        path: code-scanning-query-list.csv

.github/workflows/ruby-build.yml

@@ -1,224 +0,0 @@
-name: "Ruby: Build"
-
-on:
-  push:
-    paths:
-      - "ruby/**"
-      - .github/workflows/ruby-build.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "ruby/**"
-      - .github/workflows/ruby-build.yml
-    branches:
-      - main
-      - "rc/*"
-  workflow_dispatch:
-    inputs:
-      tag:
-        description: "Version tag to create"
-        required: false
-
-env:
-  CARGO_TERM_COLOR: always
-
-defaults:
-  run:
-    working-directory: ruby
-
-jobs:
-  build:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install GNU tar
-        if: runner.os == 'macOS'
-        run: |
-          brew install gnu-tar
-          echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ruby/target
-          key: ${{ runner.os }}-ruby-rust-cargo-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}
-      - name: Check formatting
-        run: cargo fmt --all -- --check
-      - name: Build
-        run: cargo build --verbose
-      - name: Run tests
-        run: cargo test --verbose
-      - name: Release build
-        run: cargo build --release
-      - name: Generate dbscheme
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        run: target/release/ruby-generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v3
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        with:
-          name: ruby.dbscheme
-          path: ruby/ql/lib/ruby.dbscheme
-      - uses: actions/upload-artifact@v3
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        with:
-          name: TreeSitter.qll
-          path: ruby/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v3
-        with:
-          name: extractor-${{ matrix.os }}
-          path: |
-            ruby/target/release/ruby-autobuilder
-            ruby/target/release/ruby-autobuilder.exe
-            ruby/target/release/ruby-extractor
-            ruby/target/release/ruby-extractor.exe
-          retention-days: 1
-  compile-queries:
-    runs-on: ubuntu-latest
-    env:
-      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
-    steps:
-      - uses: actions/checkout@v3
-      - name: Fetch CodeQL
-        run: |
-          LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
-          gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
-          unzip -q codeql-linux64.zip
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-      - name: Build Query Pack
-        run: |
-          codeql/codeql pack create ql/lib --output target/packs
-          codeql/codeql pack install ql/src
-          codeql/codeql pack create ql/src --output target/packs
-          PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
-          codeql/codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
-          (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
-      - uses: actions/upload-artifact@v3
-        with:
-          name: codeql-ruby-queries
-          path: |
-            ruby/target/packs/*
-          retention-days: 1
-
-  package:
-    runs-on: ubuntu-latest
-    needs: [build, compile-queries]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
-        with:
-          name: ruby.dbscheme
-          path: ruby/ruby
-      - uses: actions/download-artifact@v3
-        with:
-          name: extractor-ubuntu-latest
-          path: ruby/linux64
-      - uses: actions/download-artifact@v3
-        with:
-          name: extractor-windows-latest
-          path: ruby/win64
-      - uses: actions/download-artifact@v3
-        with:
-          name: extractor-macos-latest
-          path: ruby/osx64
-      - run: |
-          mkdir -p ruby
-          cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
-          mkdir -p ruby/tools/{linux64,osx64,win64}
-          cp linux64/ruby-autobuilder ruby/tools/linux64/autobuilder
-          cp osx64/ruby-autobuilder ruby/tools/osx64/autobuilder
-          cp win64/ruby-autobuilder.exe ruby/tools/win64/autobuilder.exe
-          cp linux64/ruby-extractor ruby/tools/linux64/extractor
-          cp osx64/ruby-extractor ruby/tools/osx64/extractor
-          cp win64/ruby-extractor.exe ruby/tools/win64/extractor.exe
-          chmod +x ruby/tools/{linux64,osx64}/{autobuilder,extractor}
-          zip -rq codeql-ruby.zip ruby
-      - uses: actions/upload-artifact@v3
-        with:
-          name: codeql-ruby-pack
-          path: ruby/codeql-ruby.zip
-          retention-days: 1
-      - uses: actions/download-artifact@v3
-        with:
-          name: codeql-ruby-queries
-          path: ruby/qlpacks
-      - run: |
-          echo '{
-            "provide": [
-            "ruby/codeql-extractor.yml",
-            "qlpacks/*/*/*/qlpack.yml"
-            ]
-          }' > .codeqlmanifest.json
-          zip -rq codeql-ruby-bundle.zip .codeqlmanifest.json ruby qlpacks
-      - uses: actions/upload-artifact@v3
-        with:
-          name: codeql-ruby-bundle
-          path: ruby/codeql-ruby-bundle.zip
-          retention-days: 1
-
-  test:
-    defaults:
-      run:
-        working-directory: ${{ github.workspace }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-
-    runs-on: ${{ matrix.os }}
-    needs: [package]
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: Shopify/example-ruby-app
-          ref: 67a0decc5eb550f3a9228eda53925c3afd40dfe9
-      - name: Fetch CodeQL
-        shell: bash
-        run: |
-          LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
-          gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql.zip "$LATEST"
-          unzip -q codeql.zip
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        working-directory: ${{ runner.temp }}
-      - name: Download Ruby bundle
-        uses: actions/download-artifact@v3
-        with:
-          name: codeql-ruby-bundle
-          path: ${{ runner.temp }}
-      - name: Unzip Ruby bundle
-        shell: bash
-        run: unzip -q -d "${{ runner.temp }}/ruby-bundle" "${{ runner.temp }}/codeql-ruby-bundle.zip"
-      - name: Prepare test files
-        shell: bash
-        run: |
-          echo "import ruby select count(File f)" > "test.ql"
-          echo "| 4 |" > "test.expected"
-          echo 'name: sample-tests
-          version: 0.0.0
-          dependencies:
-            codeql/ruby-all: 0.0.1
-          extractor: ruby
-          tests: .
-          ' > qlpack.yml
-      - name: Run QL test
-        shell: bash
-        run: |
-          "${{ runner.temp }}/codeql/codeql" test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
-      - name: Create database
-        shell: bash
-        run: |
-          "${{ runner.temp }}/codeql/codeql" database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
-      - name: Analyze database
-        shell: bash
-        run: |
-          "${{ runner.temp }}/codeql/codeql" database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/ruby-dataset-measure.yml

@@ -1,73 +0,0 @@
-name: "Ruby: Collect database stats"
-
-on:
-  push:
-    branches:
-      - main
-      - "rc/*"
-    paths:
-      - ruby/ql/lib/ruby.dbscheme
-      - .github/workflows/ruby-dataset-measure.yml
-  pull_request:
-    branches:
-      - main
-      - "rc/*"
-    paths:
-      - ruby/ql/lib/ruby.dbscheme
-      - .github/workflows/ruby-dataset-measure.yml
-  workflow_dispatch:
-
-jobs:
-  measure:
-    env:
-      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
-    strategy:
-      fail-fast: false
-      matrix:
-        repo: [rails/rails, discourse/discourse, spree/spree, ruby/ruby]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: ./.github/actions/fetch-codeql
-
-      - uses: ./ruby/actions/create-extractor-pack
-
-      - name: Checkout ${{ matrix.repo }}
-        uses: actions/checkout@v3
-        with:
-          repository: ${{ matrix.repo }}
-          path: ${{ github.workspace }}/repo
-      - name: Create database
-        run: |
-          codeql database create \
-            --search-path "${{ github.workspace }}/ruby/extractor-pack" \
-            --threads 4 \
-            --language ruby --source-root "${{ github.workspace }}/repo" \
-            "${{ runner.temp }}/database"
-      - name: Measure database
-        run: |
-          mkdir -p "stats/${{ matrix.repo }}"
-          codeql dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ruby"
-      - uses: actions/upload-artifact@v3
-        with:
-          name: measurements
-          path: stats
-          retention-days: 1
-
-  merge:
-    runs-on: ubuntu-latest
-    needs: measure
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
-        with:
-          name: measurements
-          path: stats
-      - run: |
-          python -m pip install --user lxml
-          find stats -name 'stats.xml' | sort | xargs python ruby/scripts/merge_stats.py --output ruby/ql/lib/ruby.dbscheme.stats --normalise ruby_tokeninfo
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ruby.dbscheme.stats
-          path: ruby/ql/lib/ruby.dbscheme.stats

.github/workflows/ruby-qltest.yml

@@ -1,77 +0,0 @@
-name: "Ruby: Run QL Tests"
-
-on:
-  push:
-    paths:
-      - "ruby/**"
-      - .github/workflows/ruby-qltest.yml
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "ruby/**"
-      - .github/workflows/ruby-qltest.yml
-    branches:
-      - main
-      - "rc/*"
-
-env:
-  CARGO_TERM_COLOR: always
-
-defaults:
-  run:
-    working-directory: ruby
-
-jobs:
-  qlformat:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - name: Check QL formatting
-        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
-  qlcompile:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - name: Check QL compilation
-        run: |
-          codeql query compile --check-only --threads=0 --ram 5000 --warnings=error "ql/src" "ql/examples"
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-  qlupgrade:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - name: Check DB upgrade scripts
-        run: |
-          echo >empty.trap
-          codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
-          codeql dataset upgrade testdb --additional-packs ql/lib
-          diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme
-      - name: Check DB downgrade scripts
-        run: |
-          echo >empty.trap
-          rm -rf testdb; codeql dataset import -S ql/lib/ruby.dbscheme testdb empty.trap
-          codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
-           --dbscheme=ql/lib/ruby.dbscheme --target-dbscheme=downgrades/initial/ruby.dbscheme |
-           xargs codeql execute upgrades testdb
-          diff -q testdb/ruby.dbscheme downgrades/initial/ruby.dbscheme
-  qltest:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        slice: ["1/2", "2/2"]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - uses: ./ruby/actions/create-extractor-pack
-      - name: Run QL tests
-        run: |
-          codeql test run --threads=0 --ram 5000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test
-        env:
-          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/swift-codegen.yml

@@ -1,32 +0,0 @@
-name: "Swift: Check code generation"
-
-on:
-  pull_request:
-    paths:
-      - "swift/**"
-      - .github/workflows/swift-codegen.yml
-    branches:
-      - main
-
-jobs:
-  codegen:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - uses: bazelbuild/setup-bazelisk@v2
-      - name: Run unit tests
-        run: |
-          bazel test //swift/codegen/test --test_output=errors
-      - name: Check that QL generated code was checked in
-        run: |
-          bazel run //swift/codegen
-          git add swift
-          git diff --exit-code --stat HEAD
-      - name: Generate C++ files
-        run: |
-          bazel run //swift/codegen:cppcodegen -- --cpp-output=$PWD/swift-generated-headers
-      - uses: actions/upload-artifact@v3
-        with:
-          name: swift-generated-headers
-          path: swift-generated-headers/*.h

.github/workflows/swift-qltest.yml

@@ -1,39 +0,0 @@
-name: "Swift: Run QL Tests"
-
-on:
-  pull_request:
-    paths:
-      - "swift/**"
-      - .github/workflows/swift-qltest.yml
-    branches:
-      - main
-defaults:
-  run:
-    working-directory: swift
-
-jobs:
-  qlformat:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - name: Check QL formatting
-        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
-  qltest:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os : [ubuntu-20.04, macos-latest]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - uses: bazelbuild/setup-bazelisk@v2
-      - name: Build Swift extractor
-        run: |
-          bazel run //swift:create-extractor-pack
-      - name: Run QL tests
-        run: |
-          codeql test run --threads=0 --ram 5000 --search-path "${{ github.workspace }}/swift/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition ql/test
-        env:
-          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/sync-files.yml

@@ -1,20 +0,0 @@
-name: Check synchronized files
-
-on:
-  push:
-    branches:
-      - main
-      - 'rc/*'
-  pull_request:
-    branches:
-      - main
-      - 'rc/*'
-
-jobs:
-  sync:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check synchronized files
-        run: python config/sync-files.py
-

.github/workflows/sync_files.yml

@@ -0,0 +1,21 @@
+name: Check synchronized files
+
+on:
+  push:
+    branches:
+      - main
+      - 'rc/*'
+  pull_request:
+    branches:
+      - main
+      - 'rc/*'
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Check synchronized files
+        run: scripts/sync-identical-files.py

.github/workflows/validate-change-notes.yml

@@ -1,29 +0,0 @@
-name: Validate change notes
-
-on:
-  push:
-    paths:
-      - "*/ql/*/change-notes/**/*"
-      - ".github/workflows/validate-change-notes.yml"
-    branches:
-      - main
-      - "rc/*"
-  pull_request:
-    paths:
-      - "*/ql/*/change-notes/**/*"
-      - ".github/workflows/validate-change-notes.yml"
-
-jobs:
-  check-change-note:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Setup CodeQL
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Fail if there are any errors with existing change notes
-
-        run: |
-          codeql pack release --groups cpp,csharp,java,javascript,python,ruby,-examples,-test,-experimental

.gitignore

@@ -1,57 +1,8 @@
-# editor and OS artifacts
-*~
-.DS_STORE
-*.swp
-
-# query compilation caches
+/target
+extractor-pack
+.vscode/launch.json
 .cache
-
-# qltest projects and artifacts
-*/ql/test/**/*.testproj
-*/ql/test/**/*.actual
-*/ql/test/**/go.sum
-
-# Visual studio temporaries, except a file used by QL4VS
-.vs/*
-!.vs/VSWorkspaceSettings.json
-
-# Byte-compiled python files
-*.pyc
-
-# python virtual environment folder
-.venv/
-
-# binary files created by pytest-cov
-.coverage
-
-# It's useful (though not required) to be able to unpack codeql in the ql checkout itself
-/codeql/
-
-csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
-
-# Avoid committing cached package components
+ql/test/**/*.testproj
+ql/test/**/*.actual
+ql/test/**/CONSISTENCY
 .codeql
-
-# Compiled class file
-*.class
-
-# links created by bazel
-/bazel-*
-
-# local bazel options
-/local.bazelrc
-
-# CLion project files
-/.clwb
-
-# Go build artifacts
-go/build/*
-
-# Go binaries
-go/tools/bin
-go/tools/linux64
-go/tools/osx64
-go/tools/win64
-go/tools/tokenizer.jar
-go/main
-

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "codeql"]
+	path = codeql
+	url = https://github.com/github/codeql.git

.lgtm.yml

@@ -1,31 +0,0 @@
-path_classifiers:
-  library:
-  - javascript/externs
-  - javascript/extractor/lib
-
-  test:
-  - csharp/ql/src
-  - csharp/ql/test
-  - go/ql/test
-  - javascript/extractor/parser-tests
-  - javascript/extractor/tests
-  - javascript/ql/src
-  - javascript/ql/test
-  - python/ql/src
-  - python/ql/test
-
-  example:
-  - go/ql/src
-
-queries:
-  - include: "*"
-
-extraction:
-  python:
-    python_setup:
-      version: 3
-  javascript:
-    index:
-      exclude:
-        - javascript/ql/test
-        - javascript/extractor/tests

.pre-commit-config.yaml

@@ -1,51 +0,0 @@
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
-    hooks:
-      - id: trailing-whitespace
-        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
-      - id: end-of-file-fixer
-        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
-
-  - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v13.0.1
-    hooks:
-      - id: clang-format
-        files: ^swift/.*\.(h|c|cpp)$
-
-  - repo: local
-    hooks:
-      - id: codeql-format
-        name: Fix QL file formatting
-        files: \.qll?$
-        language: system
-        entry: codeql query format --in-place
-
-      - id: sync-files
-        name: Fix files required to be identical
-        files: \.(qll?|qhelp)$
-        language: system
-        entry: python3 config/sync-files.py --latest
-        pass_filenames: false
-
-      - id: qhelp
-        name: Check query help generation
-        files: \.qhelp$
-        language: system
-        entry: python3 misc/scripts/check-qhelp.py
-
-      - id: swift-codegen
-        name: Run Swift checked in code generation
-        files: ^swift/(codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements))
-        language: system
-        entry: bazel run //swift/codegen
-        pass_filenames: false
-
-      - id: swift-codegen-unit-tests
-        name: Run Swift code generation unit tests
-        files: ^swift/codegen/.*\.py$
-        language: system
-        entry: bazel test //swift/codegen/test
-        pass_filenames: false

.vscode/.gitattributes

@@ -1 +0,0 @@
-*.json  linguist-language=JSON-with-Comments

.vscode/extensions.json

@@ -1,10 +0,0 @@
-{
-    // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations.
-    // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp
-    // List of extensions which should be recommended for users of this workspace.
-    "recommendations": [
-        "GitHub.vscode-codeql"
-    ],
-    // List of extensions recommended by VS Code that should not be recommended for users of this workspace.
-    "unwantedRecommendations": []
-}

.vscode/settings.json

@@ -1,3 +0,0 @@
-{
-    "omnisharp.autoStart": false
-}

.vscode/tasks.json

@@ -1,27 +1,14 @@
 {
-    // To run a task, select the `Terminal | Run Task...` menu option, and then select the task from
-    // the list in the dropdown, or invoke the `Tasks: Run Task` command from the command palette/
-    // To bind a keyboard shortcut to invoke a task, see https://code.visualstudio.com/docs/editor/tasks#_binding-keyboard-shortcuts-to-tasks.
-    // See https://go.microsoft.com/fwlink/?LinkId=733558
-    // for the documentation about the tasks.json format
-    "version": "2.0.0",
-    "tasks": [
-        {
-            "label": "Sync Identical Files",
-            "type": "process",
-            // Non-Windows OS will usually have Python 3 already installed at /usr/bin/python3.
-            "command": "python3",
-            "args": [
-                "config/sync-files.py",
-                "--latest"
-            ],
-            "group": "build",
-            "windows": {
-                // On Windows, use whatever Python interpreter is configured for this workspace. The default is
-                // just `python`, so if Python is already on the path, this will find it.
-                "command": "${config:python.pythonPath}",
-            },
-            "problemMatcher": []
-        }
-    ]
+	"version": "2.0.0",
+	"tasks": [
+		{
+			"type": "cargo",
+			"subcommand": "build",
+			"problemMatcher": [
+				"$rustc"
+			],
+			"group": "build",
+			"label": "Rust: cargo build"
+		}
+	]
 }

CODEOWNERS

@@ -1,44 +0,0 @@
-/cpp/ @github/codeql-c-analysis
-/csharp/ @github/codeql-csharp
-/go/ @github/codeql-go
-/java/ @github/codeql-java
-/javascript/ @github/codeql-javascript
-/python/ @github/codeql-python
-/ruby/ @github/codeql-ruby
-/swift/ @github/codeql-c
-/java/kotlin-extractor/ @github/codeql-kotlin
-/java/kotlin-explorer/ @github/codeql-kotlin
-
-# ML-powered queries
-/javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
-
-# Notify members of codeql-go about PRs to the shared data-flow library files
-/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl.qll @github/codeql-java @github/codeql-go
-/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl2.qll @github/codeql-java @github/codeql-go
-/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll @github/codeql-java @github/codeql-go
-/java/ql/src/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll @github/codeql-java @github/codeql-go
-/java/ql/src/semmle/code/java/dataflow/internal/tainttracking2/TaintTrackingImpl.qll @github/codeql-java @github/codeql-go
-
-# CodeQL tools and associated docs
-/docs/codeql-cli/ @github/codeql-cli-reviewers
-/docs/codeql-for-visual-studio-code/ @github/codeql-vscode-reviewers
-/docs/ql-language-reference/ @github/codeql-frontend-reviewers
-/docs/query-*-style-guide.md @github/codeql-analysis-reviewers
-
-# QL for QL reviewers
-/ql/ @github/codeql-ql-for-ql-reviewers
-
-# Bazel
-**/*.bazel @github/codeql-ci-reviewers
-**/*.bzl @github/codeql-ci-reviewers
-
-# Documentation etc
-/*.md @github/code-scanning-product
-/LICENSE @github/code-scanning-product
-
-# Workflows
-/.github/workflows/ @github/codeql-ci-reviewers
-/.github/workflows/go-* @github/codeql-go
-/.github/workflows/js-ml-tests.yml @github/codeql-ml-powered-queries-reviewers
-/.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
-/.github/workflows/ruby-* @github/codeql-ruby

CODE_OF_CONDUCT.md

@@ -1,126 +1,76 @@
+# Contributor Covenant Code of Conduct
+
 ## Our Pledge
 
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, religion, or sexual identity
-and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to make participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
 
 ## Our Standards
 
-Examples of behavior that contributes to a positive environment for our
-community include:
+Examples of behavior that contributes to creating a positive environment
+include:
 
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the
-  overall community
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
 
-Examples of unacceptable behavior include:
+Examples of unacceptable behavior by participants include:
 
-* The use of sexualized language or imagery, and sexual attention or
-  advances of any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
 * Public or private harassment
-* Publishing others' private information, such as a physical or email
-  address, without their explicit permission
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
 * Other conduct which could reasonably be considered inappropriate in a
   professional setting
 
-## Enforcement Responsibilities
+## Our Responsibilities
 
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
 
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
 
 ## Scope
 
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
 
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at
-opensource@github.com.
-All complaints will be reviewed and investigated promptly and fairly.
+reported by contacting the project team at opensource@github.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
 
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident.
-
-## Enforcement Guidelines
-
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series
-of actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or
-permanent ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior,  harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within
-the community.
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
 
 ## Attribution
 
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.0, available at
-https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
-
-Community Impact Guidelines were inspired by [Mozilla's code of conduct
-enforcement ladder](https://github.com/mozilla/diversity).
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
 
 [homepage]: https://www.contributor-covenant.org
 
-For answers to common questions about this code of conduct, see the FAQ at
-https://www.contributor-covenant.org/faq. Translations are available at
-https://www.contributor-covenant.org/translations.
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

CONTRIBUTING.md

@@ -1,76 +1,64 @@
-# Contributing to CodeQL
+## Contributing
 
-We welcome contributions to our CodeQL libraries and queries. Got an idea for a new check, or how to improve an existing query? Then please go ahead and open a pull request! Contributions to this project are [released](https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license) to the public under the [project's open source license](LICENSE).
+Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great.
 
-There is lots of useful documentation to help you write queries, ranging from information about query file structure to tutorials for specific target languages. For more information on the documentation available, see [CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/codeql-queries) on [codeql.github.com](https://codeql.github.com).
+Contributions to this project are [released](https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license) to the public under the [project's open source license](LICENSE).
 
-## Change notes
+Please note that this project is released with a [Contributor Code of Conduct](CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms.
 
-Any nontrivial user-visible change to a query pack or library pack should have a change note. For details on how to add a change note for your change, see [this guide](docs/change-notes.md).
+## Building and testing
 
-## Submitting a new experimental query
+See [Developer information](docs/HOWTO.md) for information on building the Ruby extractor. There is no need to rebuild the extractor if you are only developing queries.
 
-If you have an idea for a query that you would like to share with other CodeQL users, please open a pull request to add it to this repository. New queries start out in a `<language>/ql/src/experimental` directory, to which they can be merged when they meet the following requirements.
+1. Install the CodeQL CLI as described in [Getting started with the CodeQL CLI](https://codeql.github.com/docs/codeql-cli/getting-started-with-the-codeql-cli/).
 
-1. **Directory structure**
+2. Ensure that `<extraction-root>/codeql` is in your `PATH`.
 
-    There are six language-specific query directories in this repository:
+3. Clone this repository into `<extraction-root>/codeql-ruby` and change to this directory.
 
-      * C/C++: `cpp/ql/src`
-      * C#: `csharp/ql/src`
-      * Java: `java/ql/src`
-      * JavaScript: `javascript/ql/src`
-      * Python: `python/ql/src`
-      * Ruby: `ruby/ql/src`
+4. To run all tests in a directory and its subdirectories, run `codeql test run <directory>`, for example `codeql test run ql/test/query-tests/security`.
 
-    Each language-specific directory contains further subdirectories that group queries based on their `@tags` or purpose.
-    - Experimental queries and libraries are stored in the `experimental` subdirectory within each language-specific directory in the [CodeQL repository](https://github.com/github/codeql). For example, experimental Java queries and libraries are stored in `java/ql/src/experimental` and any corresponding tests in `java/ql/test/experimental`.
-    - The structure of an `experimental` subdirectory mirrors the structure of its parent directory.
-    - Select or create an appropriate directory in `experimental` based on the existing directory structure of `experimental` or its parent directory.
+6. To run an individual test, run `codeql test run <filename>`, where `<filename>` is a `.ql` or `.qlref` file, for example `codeql test run ql/test/query-tests/security/cwe-078/CommandInjection.qlref`.
 
-2. **Query metadata**
+## Adding a new query
 
-    - The query `@id` must conform to all the requirements in the [guide on query metadata](docs/query-metadata-style-guide.md#query-id-id). In particular, it must not clash with any other queries in the repository, and it must start with the appropriate language-specific prefix.
-    - The query must have a `@name` and `@description` to explain its purpose.
-    - The query must have a `@kind` and `@problem.severity` as required by CodeQL tools.
+If you have an idea for a query that you would like to share with other CodeQL users, please open a pull request to add it to this repository.
+Follow the steps below to help other users understand what your query does, and to ensure that your query is consistent with the other CodeQL queries.
 
-    For details, see the [guide on query metadata](docs/query-metadata-style-guide.md).
+1. **Consult the documentation for query writers**
 
-    Make sure the `select` statement is compatible with the query `@kind`. See [About CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/about-codeql-queries/#select-clause) on codeql.github.com.
+   There is lots of useful documentation to help you write CodeQL queries, ranging from information about query file structure to language-specific tutorials. For more information on the documentation available, see [Writing CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/) and the [CodeQL documentation](https://codeql.github.com/docs).
 
-3. **Formatting**
+2. **Format your code correctly**
 
-    - The queries and libraries must be autoformatted, for example using the "Format Document" command in [CodeQL for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-codeql-for-visual-studio-code).
+   All of the standard CodeQL queries and libraries are uniformly formatted for clarity and consistency, so we strongly recommend that all contributions follow the same formatting guidelines. If you use the CodeQL extension for Visual Studio Code, you can auto-format your query using the [Format Document command](https://help.semmle.com/codeql/codeql-for-vscode/procedures/about-codeql-for-vscode.html). For more information, see the [QL style guide](https://github.com/github/codeql/blob/main/docs/ql-style-guide.md).
 
-    If you prefer, you can either:
-    1. install the [pre-commit framework](https://pre-commit.com/) and install the configured hooks on this repo via `pre-commit install`, or
-    2. use this [pre-commit hook](misc/scripts/pre-commit) that automatically checks whether your files are correctly formatted.
+3. **Make sure your query has the correct metadata**
 
-    See the [pre-commit hook installation guide](docs/pre-commit-hook-setup.md) for instructions on the two approaches.
+   Query metadata is used to identify your query and make sure the query results are displayed properly.
+   The most important metadata to include are the `@name`, `@description`, and the `@kind`.
+   Other metadata properties (`@precision`, `@severity`, and `@tags`) are usually added after the query has been reviewed by the maintainers.
+   For more information on writing query metadata, see the [Query metadata style guide](https://github.com/github/codeql/blob/main/docs/query-metadata-style-guide.md).
 
-4. **Compilation**
+4. **Make sure the `select` statement is compatible with the query type**
 
-    - Compilation of the query and any associated libraries and tests must be resilient to future development of the [supported](docs/supported-queries.md) libraries. This means that the functionality cannot use internal libraries, cannot depend on the output of `getAQlClass`, and cannot make use of regexp matching on `toString`.
-    - The query and any associated libraries and tests must not cause any compiler warnings to be emitted (such as use of deprecated functionality or missing `override` annotations).
+   The `select` statement of your query must be compatible with the query type (determined by the `@kind` metadata property) for alert or path results to be displayed correctly in LGTM and Visual Studio Code.
+   For more information on `select` statement format, see [About CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/about-codeql-queries/#select-clause) on the [CodeQL documentation](https://codeql.github.com/docs) site.
 
-5. **Results**
+5. **Write a query help file**
 
-    - The query must have at least one true positive result on some revision of a real project.
+   Query help files explain the purpose of your query to other users. Write your query help in a `.qhelp` file and save it in the same directory as your new query.
+   For more information on writing query help, see the [Query help style guide](https://github.com/github/codeql/blob/main/docs/query-help-style-guide.md).
 
-6. **Query help files and unit tests**
+6. **Maintain backwards compatibility**
 
-	- Query help (`.qhelp`) files and unit tests are optional (but strongly encouraged!) for queries in the `experimental` directories. For more information about contributing query help files and unit tests, see [Supported CodeQL queries and libraries](docs/supported-queries.md).
+The standard CodeQL libraries must evolve in a backwards compatible manner. If any backwards incompatible changes need to be made, the existing API must first be marked as deprecated. This is done by adding a `deprecated` annotation along with a QLDoc reference to the replacement API. Only after at least one full release cycle has elapsed may the old API be removed.
 
-Experimental queries and libraries may not be actively maintained as the supported libraries evolve. They may also be changed in backwards-incompatible ways or may be removed entirely in the future without deprecation warnings.
+In addition to contributions to our standard queries and libraries, we also welcome contributions of a more experimental nature, which do not need to fulfill all the requirements listed above. See the guidelines for [experimental queries and libraries](ql/docs/experimental.md) for details.
 
-After the experimental query is merged, we welcome pull requests to improve it. Before a query can be moved out of the `experimental` subdirectory, it must satisfy [the requirements for being a supported query](docs/supported-queries.md).
+## Resources
 
-## Using your personal data
-
-If you contribute to this project, we will record your name and email address (as provided by you with your contributions) as part of the code repositories, which are public. We might also use this information to contact you in relation to your contributions, as well as in the normal course of software development. We also store records of CLA agreements signed in the past, but no longer require contributors to sign a CLA. Under GDPR legislation, we do this on the basis of our legitimate interest in creating the CodeQL product.
-
-Please do get in touch (privacy@github.com) if you have any questions about this or our data protection policies.
-
-## Bazel
-Please notice that any bazel targets and definitions in this repository are currently experimental
-and for internal use only.
+- [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)
+- [Using Pull Requests](https://help.github.com/articles/about-pull-requests/)
+- [GitHub Help](https://help.github.com)
+- [A Note About Git Commit Messages](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)

Cargo.lock

@@ -0,0 +1,661 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "adler"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
+
+[[package]]
+name = "aho-corasick"
+version = "0.7.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "ansi_term"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "ansi_term"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
+
+[[package]]
+name = "bitflags"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+
+[[package]]
+name = "byteorder"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b"
+
+[[package]]
+name = "cc"
+version = "1.0.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "chrono"
+version = "0.4.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
+dependencies = [
+ "libc",
+ "num-integer",
+ "num-traits",
+ "time",
+ "winapi",
+]
+
+[[package]]
+name = "clap"
+version = "2.33.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002"
+dependencies = [
+ "ansi_term 0.11.0",
+ "atty",
+ "bitflags",
+ "strsim",
+ "textwrap",
+ "unicode-width",
+ "vec_map",
+]
+
+[[package]]
+name = "const_fn"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6"
+
+[[package]]
+name = "crc32fast"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
+dependencies = [
+ "cfg-if",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d"
+dependencies = [
+ "cfg-if",
+ "const_fn",
+ "crossbeam-utils",
+ "lazy_static",
+ "memoffset",
+ "scopeguard",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "lazy_static",
+]
+
+[[package]]
+name = "either"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
+
+[[package]]
+name = "flate2"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0"
+dependencies = [
+ "cfg-if",
+ "crc32fast",
+ "libc",
+ "miniz_oxide",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "itoa"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "libc"
+version = "0.2.85"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ccac4b00700875e6a07c6cde370d44d32fa01c5a65cdd2fca6858c479d28bb3"
+
+[[package]]
+name = "log"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "matchers"
+version = "0.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1"
+dependencies = [
+ "regex-automata",
+]
+
+[[package]]
+name = "memchr"
+version = "2.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
+
+[[package]]
+name = "memoffset"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "miniz_oxide"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
+dependencies = [
+ "adler",
+ "autocfg",
+]
+
+[[package]]
+name = "node-types"
+version = "0.1.0"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
+dependencies = [
+ "autocfg",
+ "num-traits",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
+dependencies = [
+ "unicode-xid",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rayon"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
+dependencies = [
+ "autocfg",
+ "crossbeam-deque",
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
+ "lazy_static",
+ "num_cpus",
+]
+
+[[package]]
+name = "regex"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+ "thread_local",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4"
+dependencies = [
+ "byteorder",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581"
+
+[[package]]
+name = "ruby-autobuilder"
+version = "0.1.0"
+
+[[package]]
+name = "ruby-extractor"
+version = "0.1.0"
+dependencies = [
+ "clap",
+ "flate2",
+ "node-types",
+ "num_cpus",
+ "rayon",
+ "regex",
+ "tracing",
+ "tracing-subscriber",
+ "tree-sitter",
+ "tree-sitter-embedded-template",
+ "tree-sitter-ruby",
+]
+
+[[package]]
+name = "ruby-generator"
+version = "0.1.0"
+dependencies = [
+ "clap",
+ "node-types",
+ "tracing",
+ "tracing-subscriber",
+ "tree-sitter-embedded-template",
+ "tree-sitter-ruby",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
+
+[[package]]
+name = "scopeguard"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+
+[[package]]
+name = "serde"
+version = "1.0.123"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.123"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.61"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "sharded-slab"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79c719719ee05df97490f80a45acfc99e5a30ce98a1e4fb67aee422745ae14e3"
+dependencies = [
+ "lazy_static",
+]
+
+[[package]]
+name = "smallvec"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
+
+[[package]]
+name = "strsim"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
+
+[[package]]
+name = "syn"
+version = "1.0.60"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
+dependencies = [
+ "unicode-width",
+]
+
+[[package]]
+name = "thread_local"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
+name = "time"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
+dependencies = [
+ "libc",
+ "wasi",
+ "winapi",
+]
+
+[[package]]
+name = "tracing"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7d40a22fd029e33300d8d89a5cc8ffce18bb7c587662f54629e94c9de5487f3"
+dependencies = [
+ "cfg-if",
+ "pin-project-lite",
+ "tracing-attributes",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-attributes"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43f080ea7e4107844ef4766459426fa2d5c1ada2e47edba05dc7fa99d9629f47"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f"
+dependencies = [
+ "lazy_static",
+]
+
+[[package]]
+name = "tracing-log"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e0f8c7178e13481ff6765bd169b33e8d554c5d2bbede5e32c356194be02b9b9"
+dependencies = [
+ "lazy_static",
+ "log",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-serde"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b"
+dependencies = [
+ "serde",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-subscriber"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1fa8f0c8f4c594e4fc9debc1990deab13238077271ba84dd853d54902ee3401"
+dependencies = [
+ "ansi_term 0.12.1",
+ "chrono",
+ "lazy_static",
+ "matchers",
+ "regex",
+ "serde",
+ "serde_json",
+ "sharded-slab",
+ "smallvec",
+ "thread_local",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
+ "tracing-serde",
+]
+
+[[package]]
+name = "tree-sitter"
+version = "0.19.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad726ec26496bf4c083fff0f43d4eb3a2ad1bba305323af5ff91383c0b6ecac0"
+dependencies = [
+ "cc",
+ "regex",
+]
+
+[[package]]
+name = "tree-sitter-embedded-template"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "193fe1e3b39f71785d1890e4ac7bedd9be3d797f5db6363884415804fd516d93"
+dependencies = [
+ "cc",
+ "tree-sitter",
+]
+
+[[package]]
+name = "tree-sitter-ruby"
+version = "0.19.0"
+source = "git+https://github.com/tree-sitter/tree-sitter-ruby.git?rev=bb6a42e42b048627a74a127d3e0184c1eef01de9#bb6a42e42b048627a74a127d3e0184c1eef01de9"
+dependencies = [
+ "cc",
+ "tree-sitter",
+]
+
+[[package]]
+name = "unicode-width"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
+
+[[package]]
+name = "vec_map"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
+
+[[package]]
+name = "wasi"
+version = "0.10.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2006-2020 GitHub, Inc.
+Copyright (c) 2020-2021 GitHub
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -0,0 +1,71 @@
+all: extractor dbscheme
+
+ifeq ($(OS),Windows_NT)
+EXE = .exe
+CODEQL_PLATFORM = win64
+else
+EXE =
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Linux)
+CODEQL_PLATFORM = linux64
+endif
+ifeq ($(UNAME_S),Darwin)
+CODEQL_PLATFORM = osx64
+endif
+endif
+
+FILES=codeql-extractor.yml\
+      tools/qltest.cmd\
+      tools/index-files.sh\
+      tools/index-files.cmd\
+      tools/autobuild.sh\
+      tools/qltest.sh\
+      tools/autobuild.cmd\
+      ql/lib/ruby.dbscheme.stats\
+      ql/lib/ruby.dbscheme
+
+BIN_FILES=target/release/ruby-extractor$(EXE) target/release/ruby-autobuilder$(EXE)
+
+extractor-common:
+	rm -rf build
+	mkdir build
+	mkdir build/codeql-extractor-ruby
+	cp codeql-extractor.yml ql/lib/ruby.dbscheme ql/lib/ruby.dbscheme.stats build/codeql-extractor-ruby
+	cp -r tools build/codeql-extractor-ruby/
+
+.PHONY:	tools
+tools: $(BIN_FILES)
+	rm -rf tools/bin
+	mkdir tools/bin
+	cp -r target/release/ruby-autobuilder$(EXE) tools/bin/autobuilder$(EXE)
+	cp -r target/release/ruby-extractor$(EXE) tools/bin/extractor$(EXE)	
+
+target/release/%$(EXE):
+	cargo build --release --bin $(basename $(notdir $@))
+
+dbscheme:
+	cargo build --bin ruby-generator
+	cargo run -p ruby-generator -- --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+	codeql query format -i ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+
+.PHONY:	extractor
+extractor:	$(FILES) $(BIN_FILES)
+	rm -rf extractor-pack
+	mkdir extractor-pack
+	mkdir extractor-pack/tools
+	mkdir extractor-pack/tools/$(CODEQL_PLATFORM)
+	cp codeql-extractor.yml extractor-pack/codeql-extractor.yml
+	cp tools/qltest.cmd extractor-pack/tools/qltest.cmd
+	cp tools/index-files.sh extractor-pack/tools/index-files.sh
+	cp tools/index-files.cmd extractor-pack/tools/index-files.cmd
+	cp tools/autobuild.sh extractor-pack/tools/autobuild.sh
+	cp tools/qltest.sh extractor-pack/tools/qltest.sh
+	cp tools/autobuild.cmd extractor-pack/tools/autobuild.cmd
+	cp ql/lib/ruby.dbscheme.stats extractor-pack/ruby.dbscheme.stats
+	cp ql/lib/ruby.dbscheme extractor-pack/ruby.dbscheme
+	cp target/release/ruby-extractor$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/extractor$(EXE)
+	cp target/release/ruby-autobuilder$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/autobuilder$(EXE)
+
+test: extractor dbscheme
+	codeql pack install ql/test
+	codeql test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path . --consistency-queries ql/consistency-queries ql/test

README.md

@@ -1,30 +1,50 @@
-# CodeQL
+# Ruby analysis support for CodeQL
 
-This open source repository contains the standard CodeQL libraries and queries that power [GitHub Advanced Security](https://github.com/features/security/code) and the other application security products that [GitHub](https://github.com/features/security/) makes available to its customers worldwide.
+This open-source repository contains the extractor, CodeQL libraries, and queries that power Ruby
+support in [LGTM](https://lgtm.com) and the other CodeQL products that [GitHub](https://github.com)
+makes available to its customers worldwide.
 
-## How do I learn CodeQL and run queries?
+It contains two major components:
+  - an extractor, written in Rust, that parses Ruby source code and converts it into a database
+    that can be queried using CodeQL.
+  - static analysis libraries and queries written in [CodeQL](https://codeql.github.com/docs/) that can be
+    used to analyze such a database to find coding mistakes or security vulnerabilities.
 
-There is [extensive documentation](https://codeql.github.com/docs/) on getting started with writing CodeQL.
-You can use the [CodeQL for Visual Studio Code](https://codeql.github.com/docs/codeql-for-visual-studio-code/) extension or the [interactive query console](https://lgtm.com/help/lgtm/using-query-console) on LGTM.com (Semmle Legacy product) to try out your queries on any open source project that's currently being analyzed.
+The goal of this project is to provide comprehensive static analysis support for Ruby in CodeQL.
 
-## Contributing
+For the queries and libraries that power CodeQL support for other languages, visit [the CodeQL
+repository](https://github.com/github/codeql).
 
-We welcome contributions to our standard library and standard checks. Do you have an idea for a new check, or how to improve an existing query? Then please go ahead and open a pull request! Before you do, though, please take the time to read our [contributing guidelines](CONTRIBUTING.md). You can also consult our [style guides](https://github.com/github/codeql/tree/main/docs) to learn how to format your code for consistency and clarity, how to write query metadata, and how to write query help documentation for your query.
+## Installation
 
-## License
+Simply clone this repository. There are no external dependencies.
 
-The code in this repository is licensed under the [MIT License](LICENSE) by [GitHub](https://github.com).
+If you want to use the CodeQL extension for Visual Studio Code, import this repository into your VS
+Code workspace.
 
-The CodeQL CLI (including the CodeQL engine) is hosted in a [different repository](https://github.com/github/codeql-cli-binaries) and is [licensed separately](https://github.com/github/codeql-cli-binaries/blob/main/LICENSE.md). If you'd like to use the CodeQL CLI to analyze closed-source code, you will need a separate commercial license; please [contact us](https://github.com/enterprise/contact) for further help.
+## Usage
 
-## Visual Studio Code integration
+To analyze a Ruby codebase, either use the [CodeQL command-line
+interface](https://codeql.github.com/docs/codeql-cli/) to create a database yourself, or
+download a pre-built database from [LGTM.com](https://lgtm.com/). You can then run any of the
+queries contained in this repository either on the command line or using the VS Code extension.
 
-If you use Visual Studio Code to work in this repository, there are a few integration features to make development easier.
+Note that the [lgtm.com](https://github.com/github/codeql-ruby/tree/lgtm.com) branch of this
+repository corresponds to the version of the queries that is currently deployed on LGTM.com.
+The [main](https://github.com/github/codeql-ruby/tree/main) branch may contain changes that
+have not been deployed yet, so you may need to upgrade databases downloaded from [LGTM.com](https://lgtm.com) before
+running queries on them.
 
-### CodeQL for Visual Studio Code
+## Contributions
 
-You can install the [CodeQL for Visual Studio Code](https://marketplace.visualstudio.com/items?itemName=GitHub.vscode-codeql) extension to get syntax highlighting, IntelliSense, and code navigation for the QL language, as well as unit test support for testing CodeQL libraries and queries.
+Contributions are welcome! Please see our [contribution guidelines](CONTRIBUTING.md) and our
+[code of conduct](CODE_OF_CONDUCT.md) for details on how to participate in our community.
 
-### Tasks
+## Licensing
 
-The `.vscode/tasks.json` file defines custom tasks specific to working in this repository. To invoke one of these tasks, select the `Terminal | Run Task...` menu option, and then select the desired task from the dropdown. You can also invoke the `Tasks: Run Task` command from the command palette.
+The code in this repository is licensed under the [MIT license](LICENSE).
+
+## Resources
+
+- [Writing CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/)
+- [CodeQL documentation](https://codeql.github.com/docs/)

WORKSPACE.bazel

@@ -1,12 +0,0 @@
-# Please notice that any bazel targets and definitions in this repository are currently experimental
-# and for internal use only.
-
-workspace(name = "codeql")
-
-load("//misc/bazel:workspace.bzl", "codeql_workspace")
-
-codeql_workspace()
-
-load("//misc/bazel:workspace_deps.bzl", "codeql_workspace_deps")
-
-codeql_workspace_deps()

change-notes/1.18/analysis-cpp.md

@@ -1,28 +0,0 @@
-# Improvements to C/C++ analysis
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Upcast array used in pointer arithmetic (`cpp/upcast-array-pointer-arithmetic`) | reliability, correctness, external/cwe/cwe-119 | Finds undefined behavior caused by doing pointer arithmetic on an array of objects that has been cast to an array of a supertype. |
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Assignment where comparison was intended (`cpp/assign-where-compare-meant`) | Fewer false positive results | Results are no longer reported if the variable is not yet defined. |
-| Comparison where assignment was intended (`cpp/compare-where-assign-meant`) | More results | This query now includes results where an overloaded `operator==` is used in the wrong context. |
-| For loop variable changed in body (`cpp/loop-variable-changed`)  | Fewer false positive results | Results where the loop variable is a member of a class or struct now account for the object. |
-| Local variable hides global variable (`cpp/local-variable-hides-global-variable`) | Fewer false positive results | Results for parameters are now only reported if the name of the global variable is the same as the name of the parameter as used in the function definition (not just a function declaration). |
-| Nested loops with same variable (`cpp/nested-loops-with-same-variable`) | Fewer false positive results | Results where the loop variable is a member of a class or struct now account for the object. |
-| Self comparison (`cpp/comparison-of-identical-expressions`) | Fewer false positive results | Range checks of the form `x == (T)x` are no longer flagged unless they are guaranteed to have the same result on all platforms. |
-| Too few arguments to formatting function (`cpp/wrong-number-format-arguments`) | More precise results | This was previously known as "Wrong number of arguments to formatting function". It now focuses only on functions calls that are missing arguments, which tend to be more severe. See the next row for the new query that reports lower-severity alerts for calls with too many arguments. In addition, both queries now understand positional format arguments as supported by some libraries, and some false positive results for custom printf-like functions have been fixed.|
-| Too many arguments to formatting function (`cpp/too-many-format-arguments`) | More precise results | This new query was created by splitting the old "Wrong number of arguments to formatting function" query (see row above). It reports function calls with too many arguments.  |
-| User-controlled data in arithmetic expression (`cpp/tainted-arithmetic`) | More results | The query is extended to analyze increment, decrement, addition-assignment, and subtraction-assignment operations. |
-| Variable used in its own initializer (`cpp/use-in-own-initializer`) | Fewer false positive results | Results where a macro is used to indicate deliberate uninitialization are now excluded. |
-|Uncontrolled data in arithmetic expression (`cpp/uncontrolled-arithmetic`) | More results | The query is extended to analyze increment, decrement, addition-assignment, and subtraction-assignment operations. |
- 
-## Changes to QL libraries
-
-* The `ClassAggregateLiteral.getFieldExpr()` and `ArrayAggregateLiteral.getElementExpr()` predicates incorrectly assumed that initializer expressions appeared in the same order as the declaration order of the elements. This resulted in the association of the expressions with the wrong elements when designated initializers were used. This has been fixed.
-* Results for the `Element.getEnclosingElement()` predicate no longer included macro accesses. To explore parents and children of macro accesses, use the relevant member predicates on `MacroAccess` or `MacroInvocation`.

change-notes/1.18/analysis-csharp.md

@@ -1,60 +0,0 @@
-# Improvements to C# analysis
-
-## General improvements
-
-* Control flow analysis has been improved for `catch` clauses with filters.
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Arbitrary file write during zip extraction ("Zip Slip") (`cs/zipslip`) | security, external/cwe/cwe-022  | Identifies zip extraction routines which allow arbitrary file overwrite vulnerabilities. |
-| Local scope variable shadows member (`cs/local-shadows-member`) | maintainability, readability | Replaces the existing queries Local variable shadows class member (`cs/local-shadows-class-member`), Local variable shadows struct member (`cs/local-shadows-struct-member`), Parameter shadows class member (`cs/parameter-shadows-class-member`), and Parameter shadows struct member (`cs/parameter-shadows-struct-member`). |
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Constant condition (`cs/constant-condition`) | More results | The query has been generalized to report alerts for the old queries Null-coalescing left operand is constant (`cs/constant-null-coalescing`) and Switch selector is constant (`cs/constant-switch-selector`). |
-| Exposing internal representation (`cs/expose-implementation`) | Different results | The query has been rewritten, based on the [equivalent Java query](https://help.semmle.com/wiki/display/JAVA/Exposing+internal+representation). |
-| Local variable shadows class member (`cs/local-shadows-class-member`) | No results | The query has been replaced by the new query: Local scope variable shadows member  (`cs/local-shadows-member`). |
-| Local variable shadows struct member (`cs/local-shadows-struct-member`) | No results | The query has been replaced by the new query: Local scope variable shadows member  (`cs/local-shadows-member`). |
-| Missing Dispose call on local IDisposable (`cs/local-not-disposed`) | Fewer false positive results | The query identifies more cases where the local variable may be disposed by a library call. |
-| Nested loops with same variable (`cs/nested-loops-with-same-variable`) | Fewer false positive results | Results are no longer highlighted in nested loops that share the same condition, and do not use the variable after the inner loop. |
-| Null-coalescing left operand is constant (`cs/constant-null-coalescing`) | No results | The query has been removed, as alerts for this problem are now reported by the new query: Constant condition (`cs/constant-condition`). |
-| Parameter shadows class member (`cs/parameter-shadows-class-member`) | No results | The query has been replaced by the new query: Local scope variable shadows member  (`cs/local-shadows-member`). |
-| Parameter shadows struct member (`cs/parameter-shadows-struct-member`) | No results | The query has been replaced by the new query: Local scope variable shadows member  (`cs/local-shadows-member`). |
-| Potentially incorrect CompareTo(...) signature (`cs/wrong-compareto-signature`) | Fewer false positive results | Results are no longer highlighted in constructed types. |
-| Switch selector is constant (`cs/constant-switch-selector`) | No results | The query has been removed, as alerts for this problem are now reported by the new query: Constant condition (`cs/constant-condition`). |
-| Useless upcast (`cs/useless-upcast`) | Fewer false positive results | The query has been improved to cover more cases where upcasts may be needed. |
-
-## Changes to code extraction
-
-* The `into` part of `join` clauses is now extracted.
-* The `when` part of constant cases is now extracted.
-* Fixed a bug where `while(x is T y) ...` was not extracted correctly.
-
-## Changes to QL libraries
-
-* A new non-member predicate `mayBeDisposed()` can be used to determine if a variable is potentially disposed inside a library. It will analyze the CIL code in the library to determine this.
-* The predicate `getCondition()` has been moved from `TypeCase` to `CaseStmt`. It is now possible to get the condition of a `ConstCase` using its `getCondition()` predicate.
-* Several control flow graph entities have been renamed (the old names are deprecated but are still available in this release for backwards compatibility):
-  - `ControlFlowNode` has been renamed to `ControlFlow::Node`.
-  - `CallableEntryNode` has been renamed to `ControlFlow::Nodes::EntryNode`.
-  - `CallableExitNode` has been renamed to `ControlFlow::Nodes::ExitNode`.
-  - `ControlFlowEdgeType` has been renamed to `ControlFlow::SuccessorType`.
-  - `ControlFlowEdgeSuccessor` has been renamed to `ControlFlow::SuccessorTypes::NormalSuccessor`.
-  - `ControlFlowEdgeConditional` has been renamed to `ControlFlow::SuccessorTypes::ConditionalSuccessor`.
-  - `ControlFlowEdgeBoolean` has been renamed to `ControlFlow::SuccessorTypes::BooleanSuccessor`.
-  - `ControlFlowEdgeNullness` has been renamed to `ControlFlow::SuccessorTypes::NullnessSuccessor`.
-  - `ControlFlowEdgeMatching` has been renamed to `ControlFlow::SuccessorTypes::MatchingSuccessor`.
-  - `ControlFlowEdgeEmptiness` has been renamed to `ControlFlow::SuccessorTypes::EmptinessSuccessor`.
-  - `ControlFlowEdgeReturn` has been renamed to `ControlFlow::SuccessorTypes::ReturnSuccessor`.
-  - `ControlFlowEdgeBreak` has been renamed to `ControlFlow::SuccessorTypes::BreakSuccessor`.
-  - `ControlFlowEdgeContinue` has been renamed to `ControlFlow::SuccessorTypes::ContinueSuccessor`.
-  - `ControlFlowEdgeGotoLabel` has been renamed to `ControlFlow::SuccessorTypes::GotoLabelSuccessor`.
-  - `ControlFlowEdgeGotoCase` has been renamed to `ControlFlow::SuccessorTypes::GotoCaseSuccessor`.
-  - `ControlFlowEdgeGotoDefault` has been renamed to `ControlFlow::SuccessorTypes::GotoDefaultSuccessor`.
-  - `ControlFlowEdgeException` has been renamed to `ControlFlow::SuccessorTypes::ExceptionSuccessor`.
-
-> You should update any custom queries that use these entities to ensure that they continue working when the old names are removed in a future release.

change-notes/1.18/analysis-javascript.md

@@ -1,139 +0,0 @@
-# Improvements to JavaScript analysis
-
-## General improvements
-
-* Improved modeling of data flow through destructuring assignments may give additional results for the security queries and other queries that rely on data flow.
-
-* Improved modeling of global variables may give more true-positive results and fewer false-positive results for a variety of queries.
-
-* Improved modeling of re-export declarations may result in fewer false-positive results for a variety of queries.
-
-* Improved modeling of taint flow through array operations may give additional results for the security queries.
-
-* The taint tracking library recognizes more ways in which taint propagates. In particular, some flow through string formatters is now recognized. This may give additional results for the security queries.
-
-* The taint tracking library now recognizes additional sanitization patterns. This may give fewer false-positive results for the security queries.
-
-* Type inference for simple function calls has been improved. This may give additional results for queries that rely on type inference.
-
-* Additional heuristics have been added to `semmle.javascript.heuristics`. Add `import semmle.javascript.heuristics.all` to a query in order to activate all of the heuristics at once.
-
-* Handling of ambient TypeScript code has been improved. As a result, fewer false-positive results will be reported in `.d.ts` files.
-
-* Support for popular libraries has been improved. Consequently, queries may produce more results on code bases that use the following libraries:
- [axios](https://github.com/axios/axios), 
- [bluebird](https://bluebirdjs.com), 
- [browserid-crypto](https://github.com/mozilla/browserid-crypto), 
- [compose-function](https://github.com/stoeffel/compose-function), 
- [cookie-parser](https://github.com/expressjs/cookie-parser), 
- [cookie-session](https://github.com/expressjs/cookie-session), 
- [cross-fetch](https://github.com/lquixada/cross-fetch), 
- [crypto-js](https://github.com/https://github.com/brix/crypto-js), 
- [deep-assign](https://github.com/sindresorhus/deep-assign), 
- [deep-extend](https://github.com/unclechu/node-deep-extend), 
- [deep-merge](https://github.com/Raynos/deep-merge), 
- [deep](https://github.com/jeffomatic/deep), 
- [deepmerge](https://github.com/KyleAMathews/deepmerge), 
- [defaults-deep](https://github.com/jonschlinkert/defaults-deep), 
- [defaults](https://github.com/tmpvar/defaults), 
- [dottie](https://github.com/mickhansen/dottie.js), 
- [dotty](https://github.com/deoxxa/dotty), 
- [ent](https://github.com/substack/node-ent), 
- [entities](https://github.com/fb55/node-entities), 
- [escape-goat](https://github.com/sindresorhus/escape-goat), 
- [express-jwt](https://github.com/auth0/express-jwt), 
- [express-session](https://github.com/expressjs/session), 
- [extend-shallow](https://github.com/jonschlinkert/extend-shallow), 
- [extend](https://github.com/justmoon/node-extend), 
- [extend2](https://github.com/eggjs/extend2), 
- [fast-json-parse](https://github.com/mcollina/fast-json-parse), 
- [forge](https://github.com/digitalbazaar/forge), 
- [format-util](https://github.com/tmpfs/format-util), 
- [got](https://github.com/sindresorhus/got), 
- [global](https://github.com/Raynos/global), 
- [he](https://github.com/mathiasbynens/he), 
- [html-entities](https://github.com/mdevils/node-html-entities), 
- [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch), 
- [jquery](https://jquery.com), 
- [js-extend](https://github.com/vmattos/js-extend), 
- [json-parse-better-errors](https://github.com/zkat/json-parse-better-errors), 
- [json-parse-safe](https://github.com/joaquimserafim/json-parse-safe), 
- [json-safe-parse](https://github.com/bahamas10/node-json-safe-parse), 
- [just-compose](https://github.com/angus-c/just), 
- [just-extend](https://github.com/angus-c/just), 
- [lodash](https://lodash.com), 
- [merge-deep](https://github.com/jonschlinkert/merge-deep), 
- [merge-options](https://github.com/schnittstabil/merge-options), 
- [merge](https://github.com/yeikos/js.merge), 
- [mixin-deep](https://github.com/jonschlinkert/mixin-deep), 
- [mixin-object](https://github.com/jonschlinkert/mixin-object), 
- [MySQL2](https://github.com/sidorares/node-mysql2), 
- [node.extend](https://github.com/dreamerslab/node.extend), 
- [node-fetch](https://github.com/bitinn/node-fetch), 
- [object-assign](https://github.com/sindresorhus/object-assign), 
- [object.assign](https://github.com/ljharb/object.assign), 
- [object.defaults](https://github.com/jonschlinkert/object.defaults), 
- [parse-json](https://github.com/sindresorhus/parse-json), 
- [printf](https://github.com/adaltas/node-printf), 
- [printj](https://github.com/SheetJS/printj), 
- [q](https://documentup.com/kriskowal/q/), 
- [ramda](https://ramdajs.com), 
- [request](https://github.com/request/request), 
- [request-promise](https://github.com/request/request-promise), 
- [request-promise-any](https://github.com/request/request-promise-any), 
- [request-promise-native](https://github.com/request/request-promise-native), 
- [React Native](https://facebook.github.io/react-native/), 
- [safe-json-parse](https://github.com/Raynos/safe-json-parse), 
- [sanitize](https://github.com/pocketly/node-sanitize), 
- [sanitizer](https://github.com/theSmaw/Caja-HTML-Sanitizer), 
- [smart-extend](https://github.com/danielkalen/smart-extend), 
- [sprintf.js](https://github.com/alexei/sprintf.js), 
- [string-template](https://github.com/Matt-Esch/string-template), 
- [superagent](https://github.com/visionmedia/superagent), 
- [underscore](https://underscorejs.org), 
- [util-extend](https://github.com/isaacs/util-extend), 
- [utils-merge](https://github.com/jaredhanson/utils-merge), 
- [validator](https://github.com/chriso/validator.js), 
- [xss](https://github.com/leizongmin/js-xss), 
- [xtend](https://github.com/Raynos/xtend).
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Clear-text logging of sensitive information (`js/clear-text-logging`) | security, external/cwe/cwe-312, external/cwe/cwe-315, external/cwe/cwe-359 | Highlights logging of sensitive information, indicating a violation of [CWE-312](https://cwe.mitre.org/data/definitions/312.html). Results shown on LGTM by default. |
-| Disabling Electron webSecurity (`js/disabling-electron-websecurity`) | security, frameworks/electron | Highlights Electron browser objects that are created with the `webSecurity` property set to false. Results shown on LGTM by default. |
-| Enabling Electron allowRunningInsecureContent (`js/enabling-electron-insecure-content`) | security, frameworks/electron | Highlights Electron browser objects that are created with the `allowRunningInsecureContent` property set to true. Results shown on LGTM by default. |
-| Uncontrolled data used in remote request (`js/request-forgery`) | security, external/cwe/cwe-918 | Highlights remote requests that are built from unsanitized user input, indicating a violation of [CWE-918](https://cwe.mitre.org/data/definitions/918.html). Results are hidden on LGTM by default. |
-| Use of externally-controlled format string (`js/tainted-format-string`) | security, external/cwe/cwe-134 | Highlights format strings containing user-provided data, indicating a violation of [CWE-134](https://cwe.mitre.org/data/definitions/134.html). Results shown on LGTM by default. |
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Arguments redefined (`js/arguments-redefinition`) | Fewer results | This query previously also flagged redefinitions of `eval`. This was an oversight that is now fixed. |
-| Comparison between inconvertible types (`js/comparison-between-incompatible-types`) | Fewer results | This query now flags fewer comparisons involving parameters. The severity of this query has been revised to "warning". |
-| CORS misconfiguration for credentials transfer (`js/cors-misconfiguration-for-credentials`) | More true-positive results | This query now treats header names case-insensitively. |
-| Hard-coded credentials (`js/hardcoded-credentials`) | More true-positive results | This query now recognizes secret cryptographic keys. |
-| Incomplete string escaping or encoding (`js/incomplete-sanitization`) | New name, more true-positive results | The "Incomplete sanitization" query has been renamed to more clearly reflect its purpose. It now recognizes incomplete URL encoding and decoding. |
-| Insecure randomness (`js/insecure-randomness`) | More true-positive results | This query now recognizes secret cryptographic keys. |
-| Misleading indentation after control statement (`js/misleading-indentation-after-control-statement`) | Fewer results | This query temporarily ignores TypeScript files. |
-| Missing rate limiting (`js/missing-rate-limiting`) | More true-positive results, fewer false-positive results | This query now recognizes additional rate limiters and expensive route handlers. |
-| Omitted array element (`js/omitted-array-element`)| Fewer results | This query temporarily ignores TypeScript files. |
-| Reflected cross-site scripting (`js/reflected-xss`) | Fewer false-positive results | This query now treats header names case-insensitively. |
-| Semicolon insertion (`js/automatic-semicolon-insertion`) | Fewer results | This query temporarily ignores TypeScript files. |
-| Server-side URL redirect (`js/server-side-unvalidated-url-redirection`) | More true-positive results | This query now treats header names case-insensitively. |
-| Superfluous trailing arguments (`js/superfluous-trailing-arguments`) | Fewer false-positive results | This query now ignores calls to some empty functions. |
-| Type confusion through parameter tampering (`js/type-confusion-through-parameter-tampering`) | Fewer false-positive results | This query no longer flags emptiness checks. |
-| Uncontrolled command line (`js/command-line-injection`) | More true-positive results | This query now recognizes indirect command injection through `sh -c` and similar. |
-| Unused variable, import, function or class (`js/unused-local-variable`) | New name, fewer results | The "Unused variable" query has been renamed to reflect the fact that it highlights different kinds of unused program elements. In addition, the query no longer highlights class expressions that could be made anonymous. While technically true, these results are not interesting. |
-| Use of incompletely initialized object (`js/incomplete-object-initialization`) | Fewer results | This query now highlights the constructor instead of its erroneous `this` or `super` expressions. |
-| Useless conditional (`js/trivial-conditional`) | Fewer results | This query no longer flags uses of boolean return values and highlights fewer comparisons involving parameters. |
-
-## Changes to QL libraries
-
-* HTTP and HTTPS requests made using the Node.js `http.request` and `https.request` APIs, and the Electron `Electron.net.request` and `Electron.ClientRequest` APIs, are modeled as `RemoteFlowSources`.
-* HTTP header names are now always normalized to lower case to reflect the fact that they are case insensitive. In particular, the result of `HeaderDefinition.getAHeaderName`, and the first parameter of `HeaderDefinition.defines`, `ExplicitHeaderDefinition.definesExplicitly`, and `RouteHandler.getAResponseHeader` are now always a lower-case string.
-* New AST nodes have been added for TypeScript 2.9 and 3.0 features.
-* The class `JsonParseCall` has been deprecated. Update your queries to use `JsonParserCall` instead.
-* The handling of spread arguments in the data flow library has been changed: `DataFlow::InvokeNode.getArgument(i)` is now only defined when there is no spread argument at or before argument position `i`, and similarly `InvokeNode.getNumArgument` is only defined for invocations without spread arguments.

change-notes/1.19/analysis-cpp.md

@@ -1,53 +0,0 @@
-# Improvements to C/C++ analysis
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Cast between `HRESULT` and a Boolean type (`cpp/hresult-boolean-conversion`) | security, external/cwe/cwe-253 | Finds logic errors caused by mistakenly treating the Windows `HRESULT` type as a Boolean instead of testing it with the appropriate macros. Results are shown on LGTM by default. |
-| Cast from `char*` to `wchar_t*` (`cpp/incorrect-string-type-conversion`) | security, external/cwe/cwe-704 | Detects potentially dangerous casts from `char*` to `wchar_t*`.  Results are shown on LGTM by default. |
-| Dead code due to `goto` or `break` statement (`cpp/dead-code-goto`) | maintainability, external/cwe/cwe-561 | Detects dead code following a `goto` or `break` statement. Results are shown on LGTM by default. |
-| Inconsistent direction of for loop (`cpp/inconsistent-loop-direction`) | correctness, external/cwe/cwe-835 | Detects `for` loops where the increment and guard condition don't appear to correspond.  Results are shown on LGTM by default. |
-| Incorrect 'not' operator usage (`cpp/incorrect-not-operator-usage`) | security, external/cwe/cwe-480 | Finds uses of the logical not (`!`) operator that look like they should be bit-wise not (`~`).  Results are hidden on LGTM by default. |
-| Non-virtual destructor in base class (`cpp/virtual-destructor`) | reliability, readability, language-features | This query, `NonVirtualDestructorInBaseClass.ql`, is a replacement in LGTM for the query: No virtual destructor (`AV Rule 78.ql`). The new query ignores base classes with non-public destructors since we consider those to be adequately protected. The new version retains the query identifier, `cpp/virtual-destructor`, and results are displayed by default on LGTM. The old query is no longer run on LGTM. |
-| `NULL` application name with an unquoted path in call to `CreateProcess` (`cpp/unsafe-create-process-call`) | security, external/cwe/cwe-428 | Finds unsafe uses of the `CreateProcess` function.  Results are hidden on LGTM by default. |
-| Setting a DACL to `NULL` in a `SECURITY_DESCRIPTOR` (`cpp/unsafe-dacl-security-descriptor`) | security, external/cwe/cwe-732 | Finds code that creates world-writable objects on Windows by setting their DACL to `NULL`. Results are shown on LGTM by default. |
-
-## Changes to existing LGTM queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Comparison result is always the same (`cpp/constant-comparison`) | Fewer false positive results | Comparisons in template instantiations are now excluded from results. |
-| Empty branch of conditional (`cpp/empty-block`) | Fewer false positive results | Now recognizes commented blocks more reliably. |
-| Expression has no effect (`cpp/useless-expression`) | Fewer false positive results | Expressions in template instantiations are now excluded from results. |
-| Missing return statement (`cpp/missing-return`) | Fewer false positive results, visible by default | Improved results when a function returns a template-dependent type, or makes a non-returning call to another function. Precision increased from 'medium' to 'high' so that alerts are shown by default in LGTM. |
-| Multiplication result converted to larger type (`cpp/integer-multiplication-cast-to-long`) | Fewer false positive results | Char-typed numbers are no longer considered to be potentially large. |
-| No virtual destructor (`cpp/jsf/av-rule-78`) | No results in LGTM | This query is part of the [Joint Strike Fighter](http://www.stroustrup.com/JSF-AV-rules.pdf) suite which defines strict coding rules for air vehicles. Its query identifier has been revised to reflect this. On LGTM this query has been replaced by the similar query "Non-virtual destructor in base class", see New queries above.  The new query highlights only code that is likely to be a problem in the majority of projects. |
-| Overloaded assignment does not return 'this' (`cpp/assignment-does-not-return-this`) | Fewer false positive results | Any return statements that are unreachable are now ignored. |
-| Resource not released in destructor (`cpp/resource-not-released-in-destructor`) | Fewer false positive results | No longer highlights uses of C++ _placement new_ and results are no longer reported for resources where the destructor body is not in the snapshot database. |
-| Self comparison (`cpp/comparison-of-identical-expressions`) | Fewer false positive results | Code inside macro invocations is now excluded from the query. |
-| Static array access may cause overflow (`cpp/static-buffer-overflow`) | More correct results | Data flow to the `size` argument of a buffer operation is now checked in this query. |
-| Suspicious add with sizeof (`cpp/suspicious-add-sizeof`) | Fewer false positive results | Arithmetic with void pointers (where allowed) is now excluded from results. |
-| Unsigned comparison to zero (`cpp/unsigned-comparison-zero`) | Fewer false positive results | Comparisons in template instantiations are now excluded from results. |
-| Wrong type of arguments to formatting function (`cpp/wrong-type-format-argument`) | Fewer false positive results | False positive results involving `typedef`s have been removed.  Expected argument types are determined more accurately, especially for wide string and pointer types.  Custom (non-standard) formatting functions are also identified more accurately. |
-
-## Changes to other queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Array offset used before range check (`cpp/offset-use-before-range-check`) | More results and fewer false positive results | Now recognizes array accesses in different positions within the expression.  Code where the range is checked before and after the array access is no longer highlighted. |
-| AV Rule 164 (`cpp/jsf/av-rule-164`) | Fewer false positive results | Now accounts for explicit casts. |
-| Call to memory access function may overflow buffer (`cpp/overflow-buffer`) | More correct results | Array indexing with a negative index is now detected by this query. |
-| Global could be static (`cpp/jpl-c/limited-scope-file` and `cpp/power-of-10/global-could-be-static`)| Fewer false positive results | Variables with declarations in header files are now excluded from results. |
-| Memory is never freed (`cpp/memory-never-freed`)| Fewer false positive results | No longer highlights uses of C++ _placement new_, which returns a pointer that does not need to be freed. |
-| Negation of unsigned value (`cpp/jsf/av-rule-165`) | Fewer false positive results | Now accounts for explicit casts. |
-| Suspicious call to memset (`cpp/suspicious-call-to-memset`) | Fewer false positive results | Types involving `decltype` are now correctly compared. |
-| Variable scope too large (`cpp/jpl-c/limited-scope-function` and `cpp/power-of-10/variable-scope-too-large`) | Fewer false positive results | Variables with declarations in header files, or that are used at file scope, are now excluded from results. |
-
-## Changes to QL libraries
-
-* New hash consing library (`semmle.code.cpp.valuenumbering.HashCons`) for structural comparison of expressions. Unlike the existing library for global value numbering, this library implements a pure syntactic comparison of expressions and will equate expressions even if they may not compute the same value.
-* The `Buffer.qll` library has more conservative treatment of arrays embedded in structs. This reduces false positive results in a number of security queries, especially `cpp/overflow-buffer`.
-    * Pre-C99 encodings of _flexible array members_ are recognized more reliably.
-    * Arrays of zero size are now treated as a special case.
-* The library `semmle.code.cpp.dataflow.RecursionPrevention` is now deprecated. It was an aid for transitioning data-flow queries from 1.16 to 1.17, and it no longer has any function. Imports of this library should simply be deleted.

change-notes/1.19/analysis-csharp.md

@@ -1,44 +0,0 @@
-# Improvements to C# analysis
-
-## General improvements
-
-### Changes to the autobuilder
-
-During code extraction, when determining the target of `msbuild` or `dotnet build`, the autobuilder now looks for:
-
-* `.proj` files,
-* then `.sln` files,
-* and finally `.csproj`/`.vcxproj` files.
-
-In all three cases, when multiple files of the same type are found, the project/solution file closest to the root is used to build the project.
-
-### Control flow graph improvements
-
-* The control flow graph construction now takes simple Boolean conditions on local scope variables into account. For example, in `if (b) x = 0; if (b) x = 1;`, the control flow graph will reflect the fact that taking the `true` (resp. `false`) branch in the first condition implies taking the same branch in the second condition. In effect, the first assignment to `x` will now be identified as being dead.
-* Code that is only reachable from a constant failing assertion, such as `Debug.Assert(false)`, is considered to be unreachable.
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Uncontrolled format string (`cs/uncontrolled-format-string`) | security, external/cwe/cwe-134 | Finds data flow from remote inputs to the format string in `String.Format`. Results are shown on LGTM by default. |
-| Using a package with a known vulnerability (`cs/use-of-vulnerable-package`) | security, external/cwe/cwe-937 | Finds project build files that import packages with known vulnerabilities. Results are shown on LGTM by default. |
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Cross-site scripting (`cs/web/xss`) | More results | Finds cross-site scripting vulnerabilities in ASP.NET Core applications. |
-| Inconsistent lock sequence (`cs/inconsistent-lock-sequence`) | More results | Finds inconsistent lock sequences globally across calls. |
-| Local scope variable shadows member (`cs/local-shadows-member`) | Fewer results | Results have been removed where a constructor parameter shadows a member, because the parameter is probably used to initialize the member. |
-
-## Changes to code extraction
-
-* Arguments passed using `in` are now extracted.
-* Fixed a bug where the `dynamic` type name was not extracted correctly in certain circumstances.
-* Fixed a bug where method type signatures were extracted incorrectly in some circumstances.
-
-## Changes to QL libraries
-
-* `getArgument()` on `AccessorCall` has been improved so it now takes tuple assignments into account. For example, the argument for the implicit `value` parameter in the setter of property `P` is `0` in `(P, x) = (0, 1)`. Additionally, the argument for the `value` parameter in compound assignments is now only the expanded value, for example, in `P += 7` the argument is `P + 7` and not `7`.
-* The predicate `isInArgument()` has been added to the `AssignableAccess` class. This holds for expressions that are passed as arguments using `in`

change-notes/1.19/analysis-java.md

@@ -1,39 +0,0 @@
-# Improvements to Java analysis
-
-## General improvements
-
-Path explanations have been added to the relevant security queries. 
-Use [QL for Eclipse](https://help.semmle.com/ql-for-eclipse/Content/WebHelp/getting-started.html) 
-to run queries and explore the data flow in results.
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Arbitrary file write during archive extraction ("Zip Slip") (`java/zipslip`) | security, external/cwe/cwe-022 | Identifies extraction routines that allow arbitrary file overwrite vulnerabilities. Results are shown on LGTM by default. |
-| Missing catch of NumberFormatException (`java/uncaught-number-format-exception`) | reliability, external/cwe/cwe-248 | Finds calls to `Integer.parseInt` and similar string-to-number conversions that might raise a `NumberFormatException` without a corresponding `catch`-clause. Results are hidden on LGTM by default. |
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Array index out of bounds (`java/index-out-of-bounds`) | Fewer false positive results | Results for arrays with a length evenly divisible by 3, or some greater number, and an index being increased with a similar stride length are no longer reported. |
-| Confusing overloading of methods (`java/confusing-method-signature`) | Fewer false positive results | A correction to the inheritance relation ensures that spurious results on certain generic classes no longer occur. |
-| Query built from user-controlled sources (`java/sql-injection`) | More results | SQL injection sinks from the Spring JDBC, MyBatis, and Hibernate frameworks are now reported. |
-| Query built without neutralizing special characters (`java/concatenated-sql-query`) | More results | SQL injection sinks from the Spring JDBC, MyBatis, and Hibernate frameworks are now reported. |
-| Unreachable catch clause (`java/unreachable-catch-clause`) | Fewer false positive results | Now accounts for calls to generic methods that throw generic exceptions. |
-| Useless comparison test (`java/constant-comparison`) | Fewer false positive results | Constant comparisons guarding `java.util.ConcurrentModificationException` are no longer reported, as they are intended to always be false in the absence of API misuse. |
-
-## Changes to QL libraries
-
-* The class `ControlFlowNode` (and by extension `BasicBlock`) has until now
-  been directly equatable to `Expr` and `Stmt`.  Exploiting these equalities,
-  for example by using casts, is now deprecated, and the conversions
-  `Expr.getControlFlowNode()` and `Stmt.getControlFlowNode()` should be used
-  instead.
-* The default set of taint sources in the `FlowSources` library is extended to
-  cover parameters annotated with Spring framework annotations indicating
-  remote user input from servlets. This affects all security queries, which
-  will yield additional results on projects that use the Spring Web framework.
-* The `ParityAnalysis` library is replaced with the more general `ModulusAnalysis` library, which improves the range analysis.
-

change-notes/1.19/analysis-javascript.md

@@ -1,82 +0,0 @@
-# Improvements to JavaScript analysis
-
-## General improvements
-
-* Modeling of taint flow through array and buffer operations has been improved. This may give additional results for the security queries.
-
-* Support for AMD modules has been improved. This may give additional results for the security queries, as well as any queries that use type inference on code bases that use such modules.
-
-* Support for popular libraries has been improved. Consequently, queries may produce more results on code bases that use the following features:
-  - File system access, for example, through [fs-extra](https://github.com/jprichardson/node-fs-extra) or [globby](https://www.npmjs.com/package/globby)
-  - Outbound network access, for example, through the [fetch API](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API)
-  - The [lodash](https://lodash.com), [underscore](https://underscorejs.org/), [async](https://www.npmjs.com/package/async) and [async-es](https://www.npmjs.com/package/async-es) libraries
-
-* The taint tracking library now recognizes additional sanitization patterns. This may give fewer false-positive results for the security queries.
-
-* Type inference for function calls has been improved. This may give additional results for queries that rely on type inference.
-
-* Path explanations have been added to the relevant security queries. 
-Use [QL for Eclipse](https://help.semmle.com/ql-for-eclipse/Content/WebHelp/getting-started.html) 
-to run queries and explore the data flow in results.
-
-## New LGTM queries
-
-| **Query**                                     | **Tags**                                             | **Purpose**                                                                                                                                                                 |
-|-----------------------------------------------|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Hard-coded data interpreted as code (`js/hardcoded-data-interpreted-as-code`) | security, external/cwe/cwe-506 | Highlights locations where hard-coded data is transformed and then executed as code or interpreted as an import path, which may indicate embedded malicious code ([CWE-506](https://cwe.mitre.org/data/definitions/506.html)). Results are hidden on LGTM by default. |
-| Host header poisoning in email generation (`js/host-header-forgery-in-email-generation`)|  security, external/cwe/cwe-640 | Highlights code that generates emails with links that can be hijacked by HTTP host header poisoning, indicating a potential violation of [CWE-640](https://cwe.mitre.org/data/definitions/640.html). Results shown on LGTM by default.  |
-| Replacement of a substring with itself (`js/identity-replacement`) | correctness, security, external/cwe/cwe-116 | Highlights string replacements that replace a string with itself, which usually indicates a mistake. Results shown on LGTM by default. |
-| Stored cross-site scripting (`js/stored-xss`) | security, external/cwe/cwe-079, external/cwe/cwe-116 | Highlights uncontrolled stored values flowing into HTML content, indicating a potential violation of [CWE-079](https://cwe.mitre.org/data/definitions/79.html). Results shown on LGTM by default. |
-| Unclear precedence of nested operators (`js/unclear-operator-precedence`) | maintainability, correctness, external/cwe/cwe-783 | Highlights nested binary operators whose relative precedence is easy to misunderstand. Results shown on LGTM by default. |
-| Unneeded defensive code (`js/unneeded-defensive-code`) | correctness, external/cwe/cwe-570, external/cwe/cwe-571 | Highlights locations where defensive code is not needed. Results are shown on LGTM by default. |
-| Unsafe dynamic method access (`js/unsafe-dynamic-method-access` ) | security, external/cwe/cwe-094 | Highlights code that invokes a user-controlled method on an object with unsafe methods. Results are shown on LGTM by default. |
-| Unvalidated dynamic method access (`js/unvalidated-dynamic-method-call` ) | security, external/cwe/cwe-754 | Highlights code that invokes a user-controlled method without guarding against exceptional circumstances. Results are shown on LGTM by default. |
-| Useless assignment to property (`js/useless-assignment-to-property`) | maintainability | Highlights property assignments whose value is always overwritten. Results are shown on LGTM by default. |
-
-## Other new queries
-
-| **Query**                                     | **Tags**                                             | **Purpose**                                                                                                                                                                 |
-|-----------------------------------------------|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Enabling Node.js integration for Electron web content renderers (`js/enabling-electron-renderer-node-integration`) | security, frameworks/electron, external/cwe/cwe-094  | Highlights Electron web content renderer preferences with Node.js integration enabled, indicating a potential violation of [CWE-94](https://cwe.mitre.org/data/definitions/94.html). |
-| File data in outbound network request (`js/file-access-to-http`) | security, external/cwe/cwe-200 | Highlights locations where file data is sent in a network request, indicating a potential violation of [CWE-200](https://cwe.mitre.org/data/definitions/200.html). |
-| User-controlled data written to file (`js/http-to-file-access`) | security, external/cwe/cwe-434, external/cwe/cwe-912 | Highlights locations where user-controlled data is written to a file, indicating a potential violation of [CWE-912](https://cwe.mitre.org/data/definitions/912.html). |
-
-
-## Changes to existing queries
-
-| **Query**                      | **Expected impact**        | **Change**                                   |
-|--------------------------------|----------------------------|----------------------------------------------|
-| Ambiguous HTML id attribute (`js/duplicate-html-id`) | Lower severity | Severity revised to "warning". |
-| Clear-text logging of sensitive information (`js/clear-text-logging`) | Fewer results | Query now tracks flow more precisely. |
-| Client side cross-site scripting (`js/xss`) | More results | HTML injection in the body of an email is also highlighted. |
-| Client-side URL redirect (`js/client-side-unvalidated-url-redirection`) | Fewer false positive results | Safe redirects recognized in more cases. |
-| Conflicting HTML element attributes (`js/conflicting-html-attribute`) | Lower severity | Severity revised to "warning". |
-| Duplicate 'if' condition (`js/duplicate-condition`) | Lower severity | Severity revised to "warning". |
-| Duplicate switch case (`js/duplicate-switch-case`) | Lower severity | Severity revised to "warning". |
-| Inconsistent use of 'new' (`js/inconsistent-use-of-new`) | Simpler result presentation | Results show one call with `new` and one without. |
-| Information exposure through a stack trace (`js/stack-trace-exposure`) | More results | Cases where the entire exception object (including the stack trace) may be exposed are highlighted. |
-| Missing 'this' qualifier (`js/missing-this-qualifier`) | Fewer false positive results | Additional intentional calls to global functions are recognized. |
-| Missing CSRF middleware (`js/missing-token-validation`) | Fewer false positive results | Additional types of CSRF protection middleware are recognized. |
-| Missing variable declaration (`js/missing-variable-declaration`) | Lower severity | Severity revised to "warning". |
-| Regular expression injection (`js/regex-injection`) | Fewer false positive results | Calls to `String.prototype.search` are identified with more precision. |
-| Remote property injection (`js/remote-property-injection`) | Fewer results | No longer highlights dynamic method calls, which are now handled by two new queries: `js/unsafe-dynamic-method-access` and `js/unvalidated-dynamic-method-call`. The precision of this rule has been revised to "medium", reflecting the precision of the remaining results. Results are now hidden on LGTM by default. |
-| Self assignment (`js/redundant-assignment`) | Fewer false positive results | Self-assignments preceded by a JSDoc comment with a `@type` tag are no longer highlighted. |
-| Server-side URL redirect (`js/server-side-unvalidated-url-redirection`) | More results and fewer false positive results | More redirection calls are identified. More safe redirections are recognized and ignored. |
-| Unbound event handler receiver (`js/unbound-event-handler-receiver`) | Fewer false positive results | Additional ways that class methods can be bound are recognized. |
-| Uncontrolled data used in network request (`js/request-forgery`) | More results | Additional kinds of requests are identified. |
-| Unknown directive (`js/unknown-directive`) | Fewer false positives results | YUI compressor directives are now recognized. |
-| Unused variable, import, function or class (`js/unused-local-variable`) | Fewer false positive results and fewer results |  Imports used by the `transform-react-jsx` Babel plugin and fewer variables that may be used by `eval` calls are highlighted. Only one result is reported for an import statement with multiple unused imports. |
-| Useless assignment to local variable (`js/useless-assignment-to-local`) | Fewer false positive results | Additional ways default values can be set are recognized. |
-| Useless conditional (`js/trivial-conditional`) | More results, fewer false positive results | More types of conditional are recognized. Additional defensive coding patterns are now ignored. |
-| Whitespace contradicts operator precedence (`js/whitespace-contradicts-precedence`) | Fewer false positive results | Operators with asymmetric whitespace are no longer highlighted. |
-| Wrong use of 'this' for static method (`js/mixed-static-instance-this-access`) | More results, fewer false-positive results | Inherited methods are now identified. |
-
-## Changes to QL libraries
-
-* A `DataFlow::ParameterNode` instance now exists for all function parameters. Previously, unused parameters did not have a corresponding data-flow node.
-
-* `ReactComponent::getAThisAccess` has been renamed to `getAThisNode`. The old name is still usable but is deprecated. It no longer gets individual `this` expressions, but the `ThisNode` mentioned below.
-
-* The `DataFlow::ThisNode` class now corresponds to the implicit receiver parameter of a function, as opposed to an individual `this` expression. This means that `getALocalSource` now maps all `this` expressions within a given function to the same source. The data-flow node associated with a `ThisExpr` can no longer be cast to `DataFlow::SourceNode` or `DataFlow::ThisNode`. Using `getALocalSource` before casting, or instead of casting, is recommended.
-
-* The flow configuration framework now supports distinguishing and tracking different kinds of taint, specified by an extensible class `FlowLabel` (which can also be referred to by its alias `TaintKind`).

change-notes/1.19/analysis-python.md

@@ -1,94 +0,0 @@
-# Improvements to Python analysis
-
-## General improvements
-
-### Representation of the control flow graph
-
-The representation of the control flow graph (CFG) has been modified to better reflect the semantics of Python. As part of these changes, a new predicate `Stmt.getAnEntryNode()` has been added to make it easier to write reachability queries involving statements.
-
-#### CFG nodes removed
-
-The following statement types no longer have a CFG node for the statement itself, as their sub-expressions already contain all the
-semantically significant information:
-
-* `ExprStmt`
-* `If`
-* `Assign`
-* `Import`
-
-For example, the CFG for `if cond: foo else bar` now starts with the CFG node for `cond`.
-
-#### CFG nodes reordered
-
-For the following statement types, the CFG node for the statement now follows the CFG nodes of its sub-expressions to follow Python semantics:
-
-* `Print`
-* `TemplateWrite`
-* `ImportStar`
-
-For example the CFG for `print foo` (in Python 2) has changed from `print -> foo` to `foo -> print`, to reflect the runtime behavior.
-
-The CFG for the `with` statement has been re-ordered to more closely reflect the semantics.
-For the `with` statement:
-```python
-with cm as var:
-    body
-```
-
-* Previous CFG node order: `<with>` -> `cm` -> `var` -> `body`
-* New CFG node order: `cm` -> `<with>` -> `var` -> `body`
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Assert statement tests the truth value of a literal constant (`py/assert-literal-constant`) | reliability, correctness     | Checks whether an assert statement is testing the truth of a literal constant value. Results are hidden on LGTM by default. |
-| Flask app is run in debug mode (`py/flask-debug`) | security, external/cwe/cwe-215, external/cwe/cwe-489 | Finds instances where a Flask application is run in debug mode. Results are shown on LGTM by default. |
-| Information exposure through an exception (`py/stack-trace-exposure`) | security, external/cwe/cwe-209, external/cwe/cwe-497 | Finds instances where information about an exception may be leaked to an external user. Results are shown on LGTM by default. |
-| Jinja2 templating with autoescape=False (`py/jinja2/autoescape-false`) | security, external/cwe/cwe-079 | Finds instantiations of `jinja2.Environment` with `autoescape=False` which may allow XSS attacks. Results are hidden on LGTM by default. |
-| Request without certificate validation (`py/request-without-cert-validation`) | security, external/cwe/cwe-295 | Finds requests where certificate verification has been explicitly turned off, possibly allowing man-in-the-middle attacks. Results are hidden on LGTM by default. |
-| Use of weak cryptographic key (`py/weak-crypto-key`) | security, external/cwe/cwe-326 | Finds creation of weak cryptographic keys. Results are shown on LGTM by default. |
-
-## Changes to existing queries
-
-All taint-tracking queries now support visualization of paths in QL for Eclipse.
-Most security alerts are now visible on LGTM by default. This means that you may see results that were previously hidden for the following queries: 
-
-* Code injection (`py/code-injection`)
-* Reflected server-side cross-site scripting (`py/reflective-xss`)
-* SQL query built from user-controlled sources (`py/sql-injection`)
-* Uncontrolled data used in path expression (`py/path-injection`)
-* Uncontrolled command line (`py/command-line-injection`)
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Command injection (`py/command-line-injection`) | More results | Additional sinks in the `os`, and `popen` modules may find more results in some projects. |
-| Encoding error (`py/encoding-error`) | Better alert location | Alerts are now shown at the start of the encoding error, rather than at the top of the file. |
-| Missing call to \_\_init\_\_ during object initialization (`py/missing-call-to-init`) | Fewer false positive results | Results where it is likely that the full call chain has not been analyzed are no longer reported. |
-| URL redirection from remote source (`py/url-redirection`) | Fewer false positive results | Taint is no longer tracked from the right-hand side of binary expressions. In other words `SAFE + TAINTED` is now treated as safe. |
-
-
-## Changes to code extraction
-
-## Improved reporting of encoding errors
-
-The extractor now outputs the location of the first character that triggers an `EncodingError`. 
-Any queries that report encoding errors will now show results at the location of the character that caused the error.
-
-### Improved scalability
-
-Scaling is near linear to at least 20 CPU cores.
-
-### Improved logging
-
-* Five levels of logging are available: `CRITICAL`, `ERROR`, `WARN`, `INFO` and `DEBUG`. `WARN` is the default.
-* LGTM uses `INFO` level logging. QL tools use `WARN` level logging by default.
-* The `--verbose` flag can be specified specified multiple times to increase the logging level once per flag added.
-* The `--quiet` flag can be specified multiple times to reduce the logging level once per flag added.
-* Log lines are now in the `[SEVERITY] message` style and never overlap.
-
-## Changes to QL libraries
-
-* Taint-tracking analysis now understands HTTP requests in the `twisted` library.
-* The analysis now handles `isinstance` and `issubclass` tests involving the basic abstract base classes better. For example, the test `issubclass(list, collections.Sequence)` is now understood to be `True`
-* Taint tracking automatically tracks tainted mappings and collections, without you having to add additional taint kinds. This means that custom taints are tracked from `x` to `y` in the following flow: `l = [x]; y =l[0]`.

change-notes/1.19/extractor-javascript.md

@@ -1,27 +0,0 @@
-[[ condition: enterprise-only ]]
-
-# Improvements to JavaScript analysis
-
-## General improvements
-
-* On LGTM, files whose name ends in `.min.js` or `-min.js` are no longer extracted by default. These files usually contain minified code and any alerts in these files would be hidden by default. If you still want to extract code from these files, you can add the following filters to your `lgtm.yml` file (or add them to existing filters):
-
-```yaml
-extraction:
-  javascript:
-    index:
-      filters:
-        - include: "**/*.min.js"
-        - include: "**/*-min.js"
-```
-
-* The TypeScript compiler is now included in the LGTM Enterprise and QL command-line tools installations, and you no longer need to install it manually.
-  If you need to override the compiler version, set the `SEMMLE_TYPESCRIPT_HOME` environment variable to
-  point to an installation of the `typescript` NPM package.
-
-## Changes to code extraction
-
-The extractor now supports:
-
-* [Optional Chaining](https://github.com/tc39/proposal-optional-chaining) expressions.
-* Additional [Flow](https://flow.org/) syntax.

change-notes/1.19/support/framework-support.rst

@@ -1,61 +0,0 @@
-Frameworks and libraries
-########################
-
-The QL libraries and queries in this version have been explicitly checked against the libraries and frameworks listed below.
-
-.. pull-quote::
-
-    Tip
-    
-    If you're interested in other libraries or frameworks, you can extend the analysis to cover them. 
-    For example, by extending the data flow libraries to include data sources and sinks for additional libraries or frameworks.
-
-.. There is currently no built-in support for libraries or frameworks for C/C++.
-
-C# built-in support
-================================
-
-* ASP.Net MVC framework
-* ASP.NET Web API
-* ASP.NET Web Forms
-* ASP.NET Core
-* ASP.NET Core MVC
-* ASP.Net Core Razor
-* Razor templates
-
-
-COBOL built-in support
-===================================
-
-* Embedded SQL
-* Embedded CICS
-
-
-Java built-in support
-==================================
-
-.. csv-table:: 
-     :file: java-frameworks.csv
-     :header-rows: 1
-     :class: fullWidthTable
-     :widths: auto
-
-
-JavaScript and TypeScript built-in support
-=======================================================
-
-.. csv-table:: 
-     :file: javascript-typescript-frameworks.csv
-     :header-rows: 1
-     :class: fullWidthTable
-     :widths: auto
-
-
-Python built-in support
-====================================
-
-.. csv-table:: 
-     :file: python-frameworks.csv
-     :header-rows: 1
-     :class: fullWidthTable
-     :widths: auto

change-notes/1.19/support/java-frameworks.csv

@@ -1,10 +0,0 @@
-Name, Category
-Hibernate, Database
-iBatis / MyBatis, Database
-Java Persistence API (JPA), Database
-JDBC, Database
-Kryo deserialization, Serialization
-SnakeYaml, Serialization
-Spring JDBC, Database
-Spring MVC, Web application framework
-XStream, Serialization

change-notes/1.19/support/javascript-typescript-frameworks.csv

@@ -1,22 +0,0 @@
-Name, Category
-angularjs, HTML framework
-axios, Network communicator
-browser, Runtime environment
-electron, Runtime environment
-express, Server
-hapi, Server
-jquery, Utility library
-koa, Server
-lodash, Utility library
-mongodb, Database
-mssql, Database
-mysql, Database
-node, Runtime environment
-postgres, Database
-ramda, Utility library
-react, HTML framework
-request, Network communicator
-sequelize, Database
-sqlite3, Database
-superagent, Network communicator
-underscore, Utility library

change-notes/1.19/support/language-support.rst

@@ -1,19 +0,0 @@
-Languages and compilers
-#######################
-
-QL and LGTM version |version| support analysis of the following languages compiled by the following compilers.
-
-Note that where there are several versions or dialects of a language, the supported variants are listed.
-
-.. csv-table::
-     :file: versions-compilers.csv
-     :header-rows: 1
-     :widths: auto
-     :stub-columns: 1
-
-.. container:: footnote-group
-
-    .. [1] The best results are achieved with COBOL code that stays close to the ANSI 85 standard.  
-    .. [2] Java 11 refers to the language features used. Builds that execute on Java 6 or higher can be analyzed.
-    .. [3] JSX and Flow code, YAML, JSON, and HTML files may also be analyzed with JavaScript files. 
-    .. [4] TypeScript analysis is performed by running the JavaScript extractor with TypeScript enabled. This is the default for LGTM.   

change-notes/1.19/support/python-frameworks.csv

@@ -1,7 +0,0 @@
-Name, Category
-Django, Web application framework
-Flask, Microframework
-Pyramid, Web application framework
-Tornado, Web application framework and asynchronous networking library
-Twisted, Networking engine
-WebOb, WSGI request library

change-notes/1.19/support/versions-compilers.csv

@@ -1,16 +0,0 @@
-Language,Variants,Compilers,Extensions
-C/C++,"C89, C99, C11, C++98, C++03, C++11, C++14, C++17","Clang extensions (up to Clang 6.0)
-
-GNU extensions (up to GCC 7.3), 
-
-Microsoft extensions (up to VS 2017)","``.cpp``, ``.c++``, ``.cxx``, ``.hpp``, ``.hh``, ``.h++``, ``.hxx``, ``.c``, ``.cc``, ``.h``"
-C#,C# up to 7.2 together with .NET versions up to 4.7.1,"Microsoft Visual Studio up to 2017, 
-
-.NET Core up to 2.1","``.sln``, ``.csproj``, ``.cs``, ``.cshtml``, ``.xaml``"
-COBOL,ANSI 85 or newer [1]_.,Not applicable,"``.cbl``, ``.CBL``, ``.cpy``, ``.CPY``, ``.copy``, ``.COPY``"
-Java,"Java 11 [2]_. or lower","javac (OpenJDK and Oracle JDK)
-
-Eclipse compiler for Java (ECJ) batch compiler",``.java``
-JavaScript,ECMAScript 2018 or lower,Not applicable,"``.js``, ``.jsx``, ``.mjs``, ``.es``, ``.es6``, ``.htm``, ``.html``, ``.xhm``, ``.xhtml``, ``.vue``, ``.json`` [3]_."
-Python,"2.7, 3.5, 3.6, 3.7",Not applicable,``.py``
-TypeScript [4]_.,"2.6, 2.7, 2.8, 2.9, 3.0, 3.1",Standard TypeScript compiler,"``.ts``, ``.tsx``"

change-notes/1.20/analysis-cpp.md

@@ -1,48 +0,0 @@
-# Improvements to C/C++ analysis
-
-## General improvements
-
-* The logic for identifying auto-generated files via comments and `#line` directives has been improved.
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Array argument size mismatch (`cpp/array-arg-size-mismatch`) | reliability | Finds function calls where the size of an array being passed is smaller than the array size of the declared parameter.  Newly displayed on LGTM. |
-| Lossy function result cast (`cpp/lossy-function-result-cast`) | correctness | Finds function calls whose result type is a floating point type, which are implicitly cast to an integral type.  Newly available on LGTM but results not displayed by default. |
-| Returning stack-allocated memory (`cpp/return-stack-allocated-memory`) | reliability, external/cwe/cwe-825 | Finds functions that may return a pointer or reference to stack-allocated memory. This query existed already but has been rewritten from scratch to make the error rate low enough for use on LGTM. Results displayed by default. |
-| Use of string copy function in a condition (`cpp/string-copy-return-value-as-boolean`) | correctness | This query identifies calls to string copy functions used in conditions, where it's likely that a different function was intended to be called. Results are displayed by default on LGTM. |
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Array argument size mismatch (`cpp/array-arg-size-mismatch`) | Fewer false positive results | An exception has been added to this query for variable sized arrays. |
-| Call to memory access function may overflow buffer (`cpp/overflow-buffer`) | More correct results | This query now recognizes calls to `RtlCopyMemoryNonTemporal` and `RtlSecureZeroMemory`. |
-| Call to memory access function may overflow buffer (`cpp/overflow-buffer`) | More correct results | Calls to `fread` are now examined by this query. |
-| Lossy function result cast (`cpp/lossy-function-result-cast`) | Fewer false positive results | The whitelist of rounding functions built into this query has been expanded. |
-| Memory is never freed (`cpp/memory-never-freed`) | More correct results | Support for more Microsoft-specific memory allocation/de-allocation functions has been added. |
-| Memory may not be freed (`cpp/memory-may-not-be-freed`) | More correct results | Support for more Microsoft-specific memory allocation/de-allocation functions has been added. |
-| Mismatching new/free or malloc/delete (`cpp/new-free-mismatch`) | More correct results | Data flow through global variables for this query has been improved. |
-| 'new[]' array freed with 'delete' (`cpp/new-array-delete-mismatch`) | More correct results | Data flow through global variables for this query has been improved. |
-| 'new' object freed with 'delete[]' (`cpp/new-delete-array-mismatch`) | More correct results | Data flow through global variables for this query has been improved. |
-| Potential buffer overflow (`cpp/potential-buffer-overflow`) | Deprecated | This query has been deprecated. Use Potentially overrunning write (`cpp/overrunning-write`) and Potentially overrunning write with float to string conversion (`cpp/overrunning-write-with-float`) instead. |
-| Resource not released in destructor (`cpp/resource-not-released-in-destructor`) | Fewer false positive results | The query no longer highlights code that releases a resource via a virtual method call, function pointer, or lambda. |
-| Returning stack-allocated memory (`cpp/return-stack-allocated-memory`) | More correct results | Many more stack allocated expressions are now recognized. |
-| Suspicious add with sizeof (`cpp/suspicious-add-sizeof`) | Fewer false positive results | Pointer arithmetic on `char * const` expressions (and other variations of `char *`) are now correctly excluded from the results. |
-| Suspicious pointer scaling (`cpp/suspicious-pointer-scaling`) | Fewer false positive results | False positive results involving types that are not uniquely named in the snapshot have been fixed. |
-| Unused static variable (`cpp/unused-static-variable`) | Fewer false positive results | Variables with the attribute `unused` are now excluded from the query. |
-| Use of inherently dangerous function (`cpp/potential-buffer-overflow`) | Cleaned up | This query no longer catches uses of `gets`, and has been renamed 'Potential buffer overflow'. |
-| Use of potentially dangerous function (`cpp/potentially-dangerous-function`) | More correct results | This query now catches uses of `gets`. |
-
-
-## Changes to QL libraries
-
-* The `semmle.code.cpp.dataflow.DataFlow` library now supports _definition by reference_ via output parameters of known functions.
-    * Data flows through `memcpy` and `memmove` by default.
-    * Custom flow into or out of arguments assigned by reference can be modeled with the new class `DataFlow::DefinitionByReferenceNode`.
-    * The data flow library adds flow through library functions that are modeled in `semmle.code.cpp.models.interfaces.DataFlow`. Queries can add subclasses of `DataFlowFunction` to specify additional flow.
-* There is a new `Namespace.isInline()` predicate, which holds if the namespace was declared as `inline namespace`.
-* The `Expr.isConstant()` predicate now also holds for _address constant expressions_, which are addresses that will be constant after the program has been linked. These address constants do not have a result for `Expr.getValue()`.
-* There are new `Function.isDeclaredConstexpr()` and `Function.isConstexpr()` predicates. They can be used to tell whether a function was declared as `constexpr`, and whether it actually is `constexpr`.
-* There is a new `Variable.isConstexpr()` predicate. It can be used to tell whether a variable is `constexpr`.

change-notes/1.20/analysis-csharp.md

@@ -1,38 +0,0 @@
-# Improvements to C# analysis
-
-## Changes to existing queries
-
-| **Query**                    | **Expected impact**    | **Change**                        |
-|------------------------------|------------------------|-----------------------------------|
-| Clear text storage of sensitive information (`cs/cleartext-storage-of-sensitive-information`) | More results | Now includes data sources for user controls in `System.Windows.Forms`. |
-| Dereferenced variable is always null (`cs/dereferenced-value-is-always-null`) | Improved results | The query has been rewritten from scratch, and the analysis is now based on static single assignment (SSA) forms. Results are now shown by default in LGTM. |
-| Dereferenced variable may be null (`cs/dereferenced-value-may-be-null`) | Improved results | The query has been rewritten from scratch, and the analysis is now based on static single assignment (SSA) forms. Results are now shown by default in LGTM. |
-| Double-checked lock is not thread-safe (`cs/unsafe-double-checked-lock`) | Fewer false positive and more true positive results | No longer highlights code where the underlying field was not updated in the `lock` statement, or where the field is a `struct`. Results have been added where there are other statements inside the `lock` statement. |
-| Exposure of private information (`cs/exposure-of-sensitive-information`) | More results | Now includes data sources for user controls in `System.Windows.Forms`. |
-| Improper control of generation of code (`cs/code-injection`) | More results | Now includes data sources for user controls in `System.Windows.Forms`. |
-| Off-by-one comparison against container length (`cs/index-out-of-bounds`) | Fewer false positive results | No longer reports results when there are additional guards on the index. |
-| SQL query built from user-controlled sources (`cs/sql-injection`) | More results | Now includes data sources for user controls in `System.Windows.Forms`. |
-| Uncontrolled format string (`cs/uncontrolled-format-string`) | More results | Now includes data sources for user controls in `System.Windows.Forms`. |
-| Unused format argument (`cs/format-argument-unused`) | Fewer false positive results | No longer reports results where the format string is empty. This is often used as a default value and is not an interesting result. |
-| Use of default ToString() (`cs/call-to-object-tostring`) | Fewer false positive results | No longer reports results for `char` arrays passed to `StringBuilder.Append()`, which were incorrectly marked as using `ToString`. |
-| Use of default ToString() (`cs/call-to-object-tostring`) | Fewer results | No longer reports results when the object is an interface or an abstract class. |
-| Using a package with a known vulnerability (`cs/use-of-vulnerable-package`) | More results | This query detects packages vulnerable to CVE-2019-0657. |
-
-## Changes to code extraction
-
-* Fix extraction of `for` statements where the condition declares new variables using `is`.
-* Initializers of `stackalloc` arrays are now extracted.
-
-## Changes to QL libraries
-
-* The class `TrivialProperty` now includes library properties determined to be trivial using CIL analysis. This may increase the number of results for all queries that use data flow.
-* Taint-tracking steps have been added for the `Json.NET` package. This will improve results for queries that use taint tracking.
-* Support has been added for EntityFrameworkCore, including
-  - Stored data flow sources
-  - Sinks for SQL expressions
-  - Data flow through fields that are mapped to the database
-* Support has been added for NHibernate-Core, including
-  - Stored data flow sources
-  - Sinks for SQL expressions
-  - Data flow through fields that are mapped to the database 
-

change-notes/1.20/analysis-java.md

@@ -1,32 +0,0 @@
-# Improvements to Java analysis
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Double-checked locking is not thread-safe (`java/unsafe-double-checked-locking`) | reliability, correctness, concurrency, external/cwe/cwe-609 | Identifies wrong implementations of double-checked locking that does not use the `volatile` keyword. |
-| Race condition in double-checked locking object initialization (`java/unsafe-double-checked-locking-init-order`) | reliability, correctness, concurrency, external/cwe/cwe-609 | Identifies wrong implementations of double-checked locking that performs additional initialization after exposing the constructed object. |
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Arbitrary file write during archive extraction ("Zip Slip") (`java/zipslip`) | Fewer false positive results | Results involving a sanitization step that converts a destination `Path` to a `File` are no longer reported. |
-| Result of multiplication cast to wider type (`java/integer-multiplication-cast-to-long`) | Fewer results | Results involving conversions to `float` or `double` are no longer reported, as they were almost exclusively false positives. |
-
-## Changes to QL libraries
-
-* The deprecated library `semmle.code.java.security.DataFlow` has been removed.
-  Improved data flow libraries have been available in
-  `semmle.code.java.dataflow.DataFlow`,
-  `semmle.code.java.dataflow.TaintTracking`, and
-  `semmle.code.java.dataflow.FlowSources` since 1.16.
-* Taint tracking now includes additional default data-flow steps through
-  collections, maps, and iterators. This affects all security queries, which
-  can report more results based on such paths.
-* The `FlowSources` and `TaintTracking` libraries are extended to cover additional remote user
-  input and taint steps from the following frameworks: Guice, Protobuf, Thrift and Struts.
-  This affects all security queries, which may yield additional results on projects
-  that use these frameworks.
-
-

change-notes/1.20/analysis-javascript.md

@@ -1,67 +0,0 @@
-# Improvements to JavaScript analysis
-
-## General improvements
-
-* Support for many frameworks and libraries has been improved, in particular for:
-  - [a-sync-waterfall](https://www.npmjs.com/package/a-sync-waterfall)
-  - [Electron](https://electronjs.org)
-  - [Express](https://npmjs.org/express)
-  - [hapi](https://hapijs.com/)
-  - [js-cookie](https://github.com/js-cookie/js-cookie)
-  - [React](https://reactjs.org/)
-  - [socket.io](http://socket.io)
-  - [Vue](https://vuejs.org/)
-
-* File classification now recognizes additional generated files, for example, files from [HTML Tidy](html-tidy.org).
-
-* The taint tracking library now recognizes flow through persistent storage, class fields, and callbacks in certain cases. Handling of regular expressions has also been improved. This may give more results for the security queries.
-
-* Type inference for function calls has been improved. This may give additional results for queries that rely on type inference.
-
-* The [Closure-Library](https://github.com/google/closure-library/wiki/goog.module:-an-ES6-module-like-alternative-to-goog.provide) module system is now supported.
-
-## New queries
-
-| **Query**                                     | **Tags**                                             | **Purpose**                                                                                                                                                                 |
-|-----------------------------------------------|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Arbitrary file write during archive extraction ("Zip Slip") (`js/zipslip`) | security, external/cwe/cwe-022 | Identifies extraction routines that allow arbitrary file overwrite vulnerabilities, indicating a possible violation of [CWE-022](https://cwe.mitre.org/data/definitions/22.html). Results are shown on LGTM by default. |
-| Arrow method on Vue instance (`js/vue/arrow-method-on-vue-instance`) | reliability, frameworks/vue | Highlights arrow functions that are used as methods on Vue instances. Results are shown on LGTM by default.|
-| Cross-window communication with unrestricted target origin (`js/cross-window-information-leak`) | security, external/cwe/201, external/cwe/359 | Highlights code that sends potentially sensitive information to another window without restricting the receiver window's origin, indicating a possible violation of [CWE-201](https://cwe.mitre.org/data/definitions/201.html). Results are shown on LGTM by default. |
-| Double escaping or unescaping (`js/double-escaping`) | correctness, security, external/cwe/cwe-116 | Highlights potential double escaping or unescaping of special characters, indicating a possible violation of [CWE-116](https://cwe.mitre.org/data/definitions/116.html). Results are shown on LGTM by default. |
-| Incomplete regular expression for hostnames (`js/incomplete-hostname-regexp`) | correctness, security, external/cwe/cwe-020 |  Highlights hostname sanitizers that are likely to be incomplete, indicating a violation of [CWE-020](https://cwe.mitre.org/data/definitions/20.html). Results are shown on LGTM by default.|
-| Incomplete URL substring sanitization | correctness, security, external/cwe/cwe-020 | Highlights URL sanitizers that are likely to be incomplete, indicating a violation of [CWE-020](https://cwe.mitre.org/data/definitions/20.html). Results shown on LGTM by default. |
-| Incorrect suffix check (`js/incorrect-suffix-check`) | correctness, security, external/cwe/cwe-020 | Highlights error-prone suffix checks based on `indexOf`, indicating a potential violation of [CWE-20](https://cwe.mitre.org/data/definitions/20.html). Results are shown on LGTM by default. |
-| Loop iteration skipped due to shifting (`js/loop-iteration-skipped-due-to-shifting`) | correctness | Highlights code that removes an element from an array while iterating over it, causing the loop to skip over some elements. Results are shown on LGTM by default. |
-| Unused property (`js/unused-property`) | maintainability | Highlights properties that are unused. Results are shown on LGTM by default. |
-| Useless comparison test (`js/useless-comparison-test`) | correctness | Highlights code that is unreachable due to a numeric comparison that is always true or always false. Results are shown on LGTM by default. |
-
-## Changes to existing queries
-
-| **Query**                                  | **Expected impact**          | **Change**                                                                   |
-|--------------------------------------------|------------------------------|------------------------------------------------------------------------------|
-| Ambiguous HTML id attribute                | Fewer false positive results | This query now treats templates more conservatively. Its precision has been revised to 'high'. |
-| Assignment to exports variable             | Fewer results                | This query no longer flags code that is also flagged by the query "Useless assignment to local variable". |
-| Client-side cross-site scripting           | More true positive and fewer false positive results. | This query now recognizes WinJS functions that are vulnerable to HTML injection. It no longer flags certain safe uses of jQuery, and recognizes custom sanitizers. |
-| Hard-coded credentials                     | Fewer false positive results | This query no longer flags the empty string as a hardcoded username. |
-| Insecure randomness | More results | This query now flags insecure uses of `crypto.pseudoRandomBytes`. |
-| Reflected cross-site scripting             | Fewer false positive results. | This query now recognizes custom sanitizers. |
-| Stored cross-site scripting                | Fewer false positive results. | This query now recognizes custom sanitizers. |
-| Unbound event handler receiver (`js/unbound-event-handler-receiver`) | Fewer false positive results | Additional ways that class methods can be bound are now recognized. |
-| Uncontrolled data used in network request  | More results                 | This query now recognizes host values that are vulnerable to injection. |
-| Uncontrolled data used in path expression | Fewer false positive results | This query now recognizes the Express `root` option, which prevents path traversal. |
-| Unneeded defensive code | More true positive and fewer false positive results | This query now recognizes additional defensive code patterns. |
-| Unsafe dynamic method access              | Fewer false positive results | This query no longer flags concatenated strings as unsafe method names. |
-| Unused parameter                           | Fewer false positive results | This query no longer flags parameters with leading underscore. |
-| Unused variable, import, function or class | Fewer false positive results | This query now flags fewer variables that are implictly used by JSX elements. It no longer flags variables with a leading underscore and variables in dead code. |
-| Unvalidated dynamic method call           | More true positive results | This query now flags concatenated strings as unvalidated method names in more cases. |
-| Useless assignment to property. | Fewer false positive results | This query now treats assignments with complex right-hand sides correctly. |
-| Useless conditional | Fewer results | Additional defensive coding patterns are now ignored. |
-| Useless conditional | More true positive results | This query now flags additional uses of function call values. |
-
-## Changes to QL libraries
-
-* `DataFlow::SourceNode` is no longer an abstract class; to add new source nodes, extend `DataFlow::SourceNode::Range` instead.
-* Subclasses of `DataFlow::PropRead` are no longer automatically made source nodes; you now need to additionally define a corresponding subclass of `DataFlow::SourceNode::Range` to achieve this.
-* The deprecated libraries `semmle.javascript.DataFlow` and `semmle.javascript.dataflow.CallGraph` have been removed; they are both superseded by `semmle.javascript.dataflow.DataFlow`.
-* Overriding `DataFlow::InvokeNode.getACallee()` no longer affects the call graph seen by the interprocedural data flow libraries. To do this, the 1-argument version `getACallee(int imprecision)` can be overridden instead.
-* The predicate `DataFlow::returnedPropWrite` was intended for internal use only and is no longer available.

change-notes/1.20/analysis-python.md

@@ -1,51 +0,0 @@
-# Improvements to Python analysis
-
-## General improvements
-
-### Extractor changes
-
-The extractor now parses all Python code from a single unified grammar. This means that almost all Python code will be successfully parsed, even if mutually incompatible Python code is present in the same project. This also means that Python code for any version can be correctly parsed on a worker running any other supported version of Python. For example, Python 3.7 code is parsed correctly, even if the installed version of Python is only 3.5. This will reduce the number of syntax errors found in many projects.
-
-### Regular expression analysis improvements
-
-The Python `re` (regular expressions) module library has a couple of constants called `MULTILINE` and `VERBOSE` which determine the parsing of regular expressions. Python 3.6 changed the implementation of these constants, which resulted in false positive results for some queries. The relevant QL libraries have been updated to support both implementations which will remove false positive results from projects that use Python 3.6 and later versions.
-
-### API improvements
-
-The API has been improved to declutter the global namespace and improve discoverability and readability.
- * New predicates `ModuleObject::named(name)` and `ModuleObject.attr(name)` have been added, allowing more readable access to common objects. For example, `(any ModuleObject m | m.getName() = "sys").getAttribute("exit")` can be replaced with `ModuleObject::named("sys").attr("exit")`
- * The API for accessing builtin functions has been improved. Predicates of the form `theXXXFunction()`, such as `theLenFunction()`, have been deprecated in favor of `Object::builtin(name)`.
- * A configuration based API has been added for writing data flow and taint tracking queries. This is provided as a convenience for query authors who have written data flow or taint tracking queries for other languages, so they can use a similar format of query across multiple languages.
-
-## New queries
-
- | **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Default version of SSL/TLS may be insecure (`py/insecure-default-protocol`) | security, external/cwe/cwe-327 | Finds instances where an insecure default protocol may be used. Results are shown on LGTM by default. |
-| Incomplete regular expression for hostnames (`py/incomplete-hostname-regexp`) | security, external/cwe/cwe-020 | Finds instances where a hostname is incompletely sanitized because a regular expression contains an unescaped character. Results are shown on LGTM by default. |
-| Incomplete URL substring sanitization (`py/incomplete-url-substring-sanitization`) | security, external/cwe/cwe-020 | Finds instances where a URL is incompletely sanitized due to insufficient checks. Results are shown on LGTM by default. |
-| Insecure temporary file (`py/insecure-temporary-file`) | security, external/cwe/cwe-377 | Finds uses of the insecure and deprecated `tempfile.mktemp`, `os.tempnam`, and `os.tmpnam` functions. Results are shown on LGTM by default. |
-| Overly permissive file permissions (`py/overly-permissive-file`) | security, external/cwe/cwe-732 | Finds instances where a file is created with overly permissive permissions. Results are not shown on LGTM by default. |
-| Use of insecure SSL/TLS version (`py/insecure-protocol`) | security, external/cwe/cwe-327 | Finds instances where a known insecure protocol has been specified. Results are shown on LGTM by default. |
-
-## Changes to existing queries
-
- | **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Comparison using is when operands support \_\_eq\_\_ (`py/comparison-using-is`) | Fewer false positive results | Results where one of the objects being compared is an enum member are no longer reported. |
-| Modification of parameter with default (`py/modification-of-default-value`) | More true positive results | Instances where the mutable default value is mutated inside other functions are now also reported. |
-| Mutation of descriptor in \_\_get\_\_ or \_\_set\_\_ method (`py/mutable-descriptor`) | Fewer false positive results | Results where the mutation does not occur when calling one of the  `__get__`, `__set__` or `__delete__` methods are no longer reported. |
-| Redundant comparison (`py/redundant-comparison`) | Fewer false positive results | Results in chained comparisons are no longer reported. |
-| Unused import (`py/unused-import`) | Fewer false positive results | Results where the imported module is used in a `doctest` string are no longer reported. |
-| Unused import (`py/unused-import`) | Fewer false positive results | Results where the imported module is used in a type-hint comment are no longer reported. |
-
-
-## Changes to QL libraries
-
- * Added support for the `dill` pickle library.
- * Added support for the `bottle` web framework.
- * Added support for the `CherryPy` web framework.
- * Added support for the `falcon` web API framework.
- * Added support for the `turbogears` web framework.
-
- 

change-notes/1.20/extractor-javascript.md

@@ -1,12 +0,0 @@
-[[ condition: enterprise-only ]]
-
-# Improvements to JavaScript analysis
-
-## Changes to code extraction
-
-* Parallel extraction of JavaScript files (but not TypeScript files) on LGTM is now supported. If LGTM is configured to evaluate queries using multiple threads, then JavaScript files are also extracted using multiple threads.
-* Experimental support for [E4X](https://developer.mozilla.org/en-US/docs/Archive/Web/E4X), a legacy language extension developed by Mozilla, is available.
-* Additional [Flow](https://flow.org/) syntax is now supported.
-* [Nullish Coalescing](https://github.com/tc39/proposal-nullish-coalescing) expressions are now supported.
-* [TypeScript 3.2](https://www.typescriptlang.org/docs/handbook/release-notes/typescript-3-2.html) is now supported.
-* The TypeScript extractor now handles the control flow of logical operators and destructuring assignments more accurately.

change-notes/1.21/analysis-cpp.md

@@ -1,41 +0,0 @@
-# Improvements to C/C++ analysis
-
-## General improvements
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-| Call to alloca in a loop (`cpp/alloca-in-loop`) | reliability, correctness, external/cwe/cwe-770 | Finds calls to `alloca` in loops, which can lead to stack overflow if the number of iterations is large.  Newly displayed [on LGTM](https://lgtm.com/rules/1508831665988/). |
-| Call to function with fewer arguments than declared parameters (`cpp/too-few-arguments`) | correctness, maintainability, security | Finds all cases where the number of arguments is fewer than the number of parameters of the function, provided the function is also properly declared/defined elsewhere. Results are displayed by default [on LGTM](https://lgtm.com/rules/1508860726279/). |
-| Call to a function with one or more incompatible arguments (`cpp/mistyped-function-arguments`) | correctness, maintainability | Finds all cases where the types of arguments do not match the types of parameters of the function, provided the function is also properly declared/defined elsewhere. Results are not displayed by default [on LGTM](https://lgtm.com/rules/1508849286093/). |
-| Use of dangerous function (`cpp/dangerous-function-overflow`) | reliability, security, external/cwe/cwe-242 | Finds calls to `gets`, which does not guard against buffer overflow. These results were previously detected by the `cpp/potentially-dangerous-function` query. Results for both queries are displayed by default on LGTM. |
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Buffer not sufficient for string (`cpp/overflow-calculated`) | Fewer results | This query no longer reports results that would be found by the 'No space for zero terminator' (`cpp/no-space-for-terminator`) query. |
-| Call to function with extraneous arguments (`cpp/futile-params`) | Improved coverage | Query has been generalized to find all cases where the number of arguments exceeds the number of parameters of the function, provided the function is also properly declared/defined elsewhere. |
-| Commented-out code (`cpp/commented-out-code`) | More correct results | Commented-out preprocessor code is now detected by this query. |
-| Comparison result is always the same | Fewer false positive results | The range analysis library is now more conservative about floating point values being possibly `NaN`. |
-| Constructor with default arguments will be used as a copy constructor (`cpp/constructor-used-as-copy-constructor`) | Lowered severity and precision | The severity and precision of this query have been reduced to "warning" and "low", respectively. This coding pattern is used intentionally and safely in a number of real-world projects. Results are no longer displayed on LGTM unless you choose to display them. |
-| Dead code due to goto or break statement (`cpp/dead-code-goto`) | Fewer false positive results | Functions containing preprocessor logic are now excluded from this analysis. |
-| Memory is never freed (`cpp/memory-never-freed`) | More correct results | Support added for more Microsoft-specific allocation functions, including `LocalAlloc`, `GlobalAlloc`, `HeapAlloc` and `CoTaskMemAlloc`. |
-| Memory may not be freed (`cpp/memory-may-not-be-freed`) | More correct results | Support added for more Microsoft-specific allocation functions, including `LocalAlloc`, `GlobalAlloc`, `HeapAlloc` and `CoTaskMemAlloc`. |
-| Mismatching new/free or malloc/delete (`cpp/new-free-mismatch`) | Fewer false positive results | Fixed an issue where functions were being identified as allocation functions inappropriately.  This correction also affects `cpp/new-array-delete-mismatch` and `cpp/new-delete-array-mismatch`. |
-| No space for zero terminator (`cpp/no-space-for-terminator`) | More correct results | This query now detects calls to `std::malloc`. |
-| Overflow in uncontrolled allocation size (`cpp/uncontrolled-allocation-size`) | More correct results | This query has been reworked so that it can find a wider variety of results. |
-| Resource not released in destructor (`cpp/resource-not-released-in-destructor`) | Fewer false positive results | Resource allocation and deallocation functions are now determined more accurately. |
-| Use of potentially dangerous function | More correct results | Calls to `localtime`, `ctime` and `asctime` are now detected by this query. |
-| Wrong type of arguments to formatting function (`cpp/wrong-type-format-argument`) | More correct results and fewer false positive results | This query now understands non-standard uses of `%L`. In addition, it more accurately identifies wide and non-wide string/character format arguments on different platforms. |
-| Use of potentially dangerous function (`cpp/potentially-dangerous-function`) | Fewer results | Results relating to the standard library `gets` function have been moved into a new query (`cpp/dangerous-function-overflow`). |
-
-## Changes to QL libraries
-- The predicate `Declaration.hasGlobalName` now only holds for declarations that are not nested in a class. For example, it no longer holds for a member function `MyClass::myFunction` or a constructor `MyClass::MyClass`, whereas previously it would classify those two declarations as global names.
-- In class `Declaration`, predicates `getQualifiedName/0` and `hasQualifiedName/1` are no longer recommended for finding functions by name. Instead, use `hasGlobalName/1` and the new `hasQualifiedName/2` and `hasQualifiedName/3` predicates. This improves performance and identifies names involving templates and inline namespaces more reliably.
-- Additional support for definition by reference has been added to the `semmle.code.cpp.dataflow.TaintTracking` library, including:
-    - Taint-specific edges for functions modeled in `semmle.code.cpp.models.interfaces.DataFlow`.
-    - Flow through library functions that are modeled in `semmle.code.cpp.models.interfaces.Taint`. Queries can add subclasses of `TaintFunction` to specify additional flow.
-- There is a new `FoldExpr` class, representing C++17 fold expressions.
-- The member predicates `DeclarationEntry.getUnspecifiedType`, `Expr.getUnspecifiedType`, and `Variable.getUnspecifiedType` have been added. These should be preferred over the existing `getUnderlyingType` predicates.

change-notes/1.21/analysis-csharp.md

@@ -1,49 +0,0 @@
-# Improvements to C# analysis
-
-## General improvements
-
-C# analysis now supports the extraction and analysis of many C# 8 features. For details see [Changes to code extraction](#changes-to-code-extraction) and [Changes to QL libraries](#changes-to-ql-libraries) below.
-
-## New queries
-
-| **Query**                                     | **Tags**                                             | **Purpose**                                                                                                                                                                 |
-|-----------------------------------------------|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Thread-unsafe capturing of an ICryptoTransform object (`cs/thread-unsafe-icryptotransform-captured-in-lambda`) | concurrency, security, external/cwe/cwe-362 | Highlights instances of classes where a field of type `System.Security.Cryptography.ICryptoTransform` is captured by a lambda, and appears to be used in a thread initialization method. Results are not shown on [LGTM](https://lgtm.com/rules/1508141845995/) by default. |
-
-## Changes to existing queries
-
-| **Query**                    | **Expected impact**    | **Change**                        |
-|------------------------------|------------------------|-----------------------------------|
-| Constant condition (`cs/constant-condition`) | Fewer false positive results | The query now ignores code where the `null` value is in a conditional expression on the left hand side of a null-coalescing expression. For example, in `(a ? b : null) ?? c`, `null` is not considered to be a constant condition. |
-| Thread-unsafe use of a static ICryptoTransform field (`cs/thread-unsafe-icryptotransform-field-in-class`) | Fewer false positive results | The criteria for a result has changed to include nested properties, nested fields, and collections. The format of the alert message has changed to highlight the static field. The query name has been updated. |
-| Useless upcast (`cs/useless-upcast`) | Fewer false positive results | The query now ignores code where the upcast is used to disambiguate the target of a constructor call. |
-
-## Changes to code extraction
-
-* The following C# 8 features are now extracted:
-    - Range expressions
-    - Recursive patterns
-    - Using declaration statements
-    - `static` modifiers on local functions
-    - Null-coalescing assignment expressions
-
-* The `unmanaged` type parameter constraint is also now extracted.
-
-## Changes to QL libraries
-
-* The class `Attribute` has two new predicates: `getConstructorArgument()` and `getNamedArgument()`. The first predicate returns arguments to the underlying constructor call and the second returns named arguments for initializing fields and properties.
-* The class `TypeParameterConstraints` has a new predicate `hasUnmanagedTypeConstraint()`. This shows whether the type parameter has the `unmanaged` constraint.
-* The following QL classes have been added to model C# 8 features:
-    - Class `AssignCoalesceExpr` models null-coalescing assignment, for example `x ??= y`
-    - Class `IndexExpr` models from-end index expressions, for example `^1`
-    - Class `PatternExpr` is an `Expr` that appears in a pattern. It has the new subclasses `DiscardPatternExpr`, `LabeledPatternExpr`, `RecursivePatternExpr`, `TypeAccessPatternExpr`, `TypePatternExpr`, and `VariablePatternExpr`.
-    - Class `PatternMatch` models a pattern being matched. It has the subclasses `Case` and `IsExpr`.
-    - Class `PositionalPatternExpr` models position patterns, for example `(int x, int y)`
-    - Class `PropertyPatternExpr` models property patterns, for example `Length: int len`
-    - Class `RangeExpr` models range expressions, for example `1..^1`
-    - Class `SwitchCaseExpr` models the arm of a switch expression, for example `(false, false) => true`
-    - Class `SwitchExpr` models `switch` expressions, for example `(a, b) switch { ... }`
-    - Classes `IsConstantExpr`, `IsTypeExpr` and `IsPatternExpr` are deprecated in favour of `IsExpr`
-    - Class `Switch` models both `SwitchExpr` and `SwitchStmt`
-    - Class `Case` models both `CaseStmt` and `SwitchCaseExpr`
-    - Class `UsingStmt` models both `UsingBlockStmt` and `UsingDeclStmt`

change-notes/1.21/analysis-java.md

@@ -1,26 +0,0 @@
-# Improvements to Java analysis
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Implicit conversion from array to string (`java/print-array`) | Fewer false positive results | Results in slf4j logging calls are no longer reported as slf4j supports array printing. |
-| Result of multiplication cast to wider type (`java/integer-multiplication-cast-to-long`) | Fewer false positive results | Range analysis is now used to exclude results involving multiplication of small values that cannot overflow. |
-
-## Changes to QL libraries
-
-* The `Guards` library has been extended to account for method calls that check
-  conditions by conditionally throwing an exception. This includes the
-  `checkArgument` and `checkState` methods in
-  `com.google.common.base.Preconditions`, the `isTrue` and `validState` methods
-  in `org.apache.commons.lang3.Validate`, as well as any similar custom
-  methods. This means that more guards are recognized which improves the precision of a number of queries including `java/index-out-of-bounds`,
-  `java/dereferenced-value-may-be-null`, and `java/useless-null-check`.
-* The default sanitizer in taint tracking has been made more precise. The
-  sanitizer works by looking for guards that inspect tainted strings. It
-  previously worked at the level of individual variables. Now it
-  uses the `Guards` library, such that only guarded variable accesses are
-  sanitized. This may give additional results for security queries.
-* Spring framework support now takes into account additional
-  annotations that indicate remote user input. This affects all security
-  queries, which may give additional results.

change-notes/1.21/analysis-javascript.md

@@ -1,62 +0,0 @@
-# Improvements to JavaScript analysis
-
-## General improvements
-
-* Support for the following frameworks and libraries has been improved:
-  - [koa](https://github.com/koajs/koa)
-  - [socket.io](http://socket.io)
-  - [Node.js](http://nodejs.org)
-  - [Firebase](https://firebase.google.com/)
-  - [Express](https://expressjs.com/)
-  - [shelljs](https://www.npmjs.com/package/shelljs)
-  - [cheerio](https://www.npmjs.com/package/cheerio)
-
-* The security queries now track data flow through Base64 decoders such as the Node.js `Buffer` class, the DOM function `atob`, and a number of npm packages including [`abab`](https://www.npmjs.com/package/abab), [`atob`](https://www.npmjs.com/package/atob), [`btoa`](https://www.npmjs.com/package/btoa), [`base-64`](https://www.npmjs.com/package/base-64), [`js-base64`](https://www.npmjs.com/package/js-base64), [`Base64.js`](https://www.npmjs.com/package/Base64) and [`base64-js`](https://www.npmjs.com/package/base64-js).
-
-* The security queries now track data flow through exceptions.
-
-* The security queries now treat comparisons with symbolic constants as sanitizers, resulting in fewer false positive results.
-
-* TypeScript 3.5 is now supported.
-
-* On LGTM, TypeScript projects now have static type information extracted by default, resulting in more security results.
-  Users of the command-line tools must still pass `--typescript-full` to the extractor to enable this.
-
-
-## New queries
-
-| **Query**                                     | **Tags**                                             | **Purpose**                                                                                                                                                                 |
-|-----------------------------------------------|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Missing regular expression anchor (`js/regex/missing-regexp-anchor`) | correctness, security, external/cwe/cwe-20 | Highlights regular expression patterns that may be missing an anchor, indicating a possible violation of [CWE-20](https://cwe.mitre.org/data/definitions/20.html). Results are not shown on LGTM by default. |
-| Prototype pollution (`js/prototype-pollution`)    | security, external/cwe-250, external/cwe-400 | Highlights code that allows an attacker to modify a built-in prototype object through an unsanitized recursive merge function. Results are not shown on [LGTM](https://lgtm.com/rules/1508857356317/) by default. |
-
-## Changes to existing queries
-
-| **Query**                      | **Expected impact**          | **Change**                                                                |
-|--------------------------------|------------------------------|---------------------------------------------------------------------------|
-| Arbitrary file write during zip extraction ("Zip Slip") | More results | This rule now considers more libraries, including tar as well as zip. |
-| Client-side URL redirect       | More results and fewer false-positive results | This rule now recognizes additional uses of the document URL. It also treats URLs as safe in more cases where the hostname cannot be tampered with. |
-| Double escaping or unescaping | More results | This rule now considers the flow of regular expressions literals. |
-| Expression has no effect       | Fewer false-positive results | This rule now treats uses of `Object.defineProperty` more conservatively. |
-| Incomplete regular expression for hostnames | More results | This rule now tracks regular expressions for host names further. |
-| Incomplete string escaping or encoding | More results | This rule now considers the flow of regular expressions literals, and it no longer flags the removal of trailing newlines. |
-| Incorrect suffix check | Fewer false-positive results | This rule now recognizes valid checks in more cases. |
-| Password in configuration file | Fewer false positive results | This query now excludes passwords that are inserted into the configuration file using a templating mechanism or read from environment variables. Results are no longer shown on LGTM by default. |
-| Replacement of a substring with itself | More results | This rule now considers the flow of regular expressions literals. |
-| Server-side URL redirect       | Fewer false-positive results | This rule now treats URLs as safe in more cases where the hostname cannot be tampered with. |
-| Tainted path | More results and fewer false-positive results | This rule now analyzes path manipulation code more precisely. |
-| Type confusion through parameter tampering | Fewer false-positive results | This rule now recognizes additional emptiness checks. |
-| Useless assignment to property | Fewer false-positive results | This rule now ignores reads of additional getters. |
-| Unreachable statement | Unreachable throws no longer give an alert | This ignores unreachable throws, as they could be intentional (for example, to placate the TS compiler). |
-
-
-## Changes to QL libraries
-
-* `RegExpLiteral` is now a `DataFlow::SourceNode`.
-* `JSDocTypeExpr` now has source locations and is a subclass of `Locatable` and `TypeAnnotation`.
-* The two-parameter versions of predicate `isBarrier` in `DataFlow::Configuration` and of predicate `isSanitizer` in `TaintTracking::Configuration` have been renamed to `isBarrierEdge` and `isSanitizerEdge`, respectively. The old names are maintained for backwards-compatibility in this version, but will be deprecated in the next version and subsequently removed.
-* Various predicates named `getTypeAnnotation()` now return `TypeAnnotation` instead of `TypeExpr`.
-  In rare cases, this may cause compilation errors in existing code. Cast the result to `TypeExpr` if this happens.
-* The `getALabel` predicate in `LabeledBarrierGuardNode` and `LabeledSanitizerGuardNode`
-  has been deprecated and overriding it no longer has any effect.
-  Instead use the 3-parameter version of `blocks` or `sanitizes`.

change-notes/1.21/analysis-python.md

@@ -1,49 +0,0 @@
-# Improvements to Python analysis
-
-
-## General improvements
-
-Points-to analysis has been re-implemented to support more language features and provide better reachability analysis.
-The new implementation adds the following new features:
-
-* Non-local tracking of bound methods and instances of `super()`
-* Superior analysis of conditionals and thus improved reachability analysis.
-* Superior modelling of descriptors, for example, classmethods and staticmethods.
-* Superior tracking of values through parameters, especially `*` arguments.
-
-A new object API has been provided to complement the new points-to implementation.
-A new class `Value` replaces the old `Object` class. The `Value` class has a simpler and more consistent API compared to `Object`.
-Some of the functionality of `FunctionObject` and `ClassObject` has been added to `Value` to reduce the number of casts to more specific classes.
-For example, the QL to find calls to `os.path.open` has changed from
-`ModuleObject::named("os").attr("path").(ModuleObject).attr("join").(FunctionObject).getACall()`
-to
-`Value::called("os.path.join").getACall()`
-
-The old API is now deprecated, but will be continued to be supported for at least another year.
-
-### Impact on existing queries.
-
-As points-to analysis underpins many queries, and provides the call-graph and reachability analysis required for taint-tracking, the results of many queries may change.
-
-The improved reachability analysis and non-local tracking of bound methods may identify new results.
-The increased precision in tracking of values through `*` arguments may remove false positive results.
-
-Overall the number of true positive results should increase and the number false negative results should decline.
-We welcome feedback on the new implementation, particularly any surprising changes in results.
-
-## New queries
-
-| **Query** | **Tags** | **Purpose** |
-|-----------|----------|-------------|
-| Accepting unknown SSH host keys when using Paramiko (`py/paramiko-missing-host-key-validation`) | security, external/cwe/cwe-295 | Finds instances where Paramiko is configured to accept unknown host keys. Results are shown [on LGTM](https://lgtm.com/rules/1508297729270/) by default. |
-| Pythagorean calculation with sub-optimal numerics (`py/pythagorean`) | accuracy | Finds instances of hypotenuse calculation using `math.sqrt` instead of `math.hypot`. Results are not shown on LGTM by default. |
-| Use of 'return' or 'yield' outside a function (`py/return-or-yield-outside-function`) | reliability, correctness | Finds instances where `return`, `yield`, and `yield from` are used outside a function. Results are not shown on LGTM by default. |
-
-## Changes to code extraction
-
-* String literals as expressions within literal string interpolation (f-strings) are now handled correctly.
-
-* The Python extractor now handles invalid input more robustly. In particular, it exits gracefully when:
-
-    * A non-existent file or directory is specified using the `--path` option, or as a file name.
-    * An invalid number is specified for the `--max-procs` option.

change-notes/1.21/extractor-javascript.md

@@ -1,17 +0,0 @@
-[[ condition: enterprise-only ]]
-
-# Improvements to JavaScript analysis
-
-## Changes to code extraction
-
-* Custom file types can now be specified using the `filetypes` property in the `extraction/javascript/index` section of `lgtm.yml`. The property should be a map from file extensions (including the dot) to file types. Valid file types are `html`, `js`, `json`, `typescript`, `xml` and `yaml`.
-
-* ECMAScript 2019 support is now enabled by default.
-
-* On LGTM, JavaScript extraction for projects that do not contain any JavaScript or TypeScript code will now fail, even if the project contains other file types (such as HTML or YAML) recognized by the JavaScript extractor.
-
-* XML files can now be extracted on LGTM. To enable XML extraction, set the `xml_mode` property in the `extraction/javascript/index` section of your `lgtm.yml` file to `all`. The default value of this property is `disabled`, meaning that XML files will not be extracted. (Note, that the `xml_mode` property does not apply to files that you map to the `xml` file type using the `filetypes` property. LGTM will always extract these files.)
-
-* YAML files are now extracted by default on LGTM. If required, you can specify exclusion filters in your `lgtm.yml` file to override this behavior.
-
-For detailed information about customizing LGTM extraction, see [JavaScript extraction](https://help.semmle.com/lgtm-enterprise/user/help/javascript-extraction.html).

change-notes/1.22/analysis-cpp.md

@@ -1,42 +0,0 @@
-# Improvements to C/C++ analysis
-
-The following changes in version 1.22 affect C/C++ analysis in all applications.
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-| Call to alloca in a loop (`cpp/alloca-in-loop`) | Fewer false positive results | The query no longer highlights code where the stack allocation could not be reached multiple times in the loop, typically due to a `break` or `return` statement. |
-| Continue statement that does not continue (`cpp/continue-in-false-loop`) | Fewer false positive results | Analysis is now restricted to `do`-`while` loops. This query is now run and displayed by default on LGTM. |
-| Expression has no effect (`cpp/useless-expression`) | Fewer false positive results | Calls to functions with the `weak` attribute are no longer considered to be side-effect free, because they could be overridden with a different implementation at link time. |
-| No space for zero terminator (`cpp/no-space-for-terminator`) | Fewer false positive results | False positive results for strings that are not null-terminated have been excluded. |
-| Non-constant format string (`cpp/non-constant-format`) | Fewer false positive results | The query was rewritten using the taint-tracking library. |
-| Sign check of bitwise operation (`cpp/bitwise-sign-check`) | Fewer false positive and more true positive results | The query now understands the direction of each comparison, making it more accurate. |
-| Suspicious pointer scaling (`cpp/suspicious-pointer-scaling`) | Lower precision | The precision of this query has been reduced to "medium". This coding pattern is used intentionally and safely in a number of real-world projects. Results are no longer displayed on LGTM unless you choose to display them. |
-| Variable used in its own initializer (`cpp/use-in-own-initializer`) | Fewer false positive results | False positive results for constant variables with the same name in different namespaces have been removed. |
-
-## Changes to QL libraries
-
-- The data flow library (`semmle.code.cpp.dataflow.DataFlow`) has had the
-  following improvements, all of which benefit the taint tracking library
-  (`semmle.code.cpp.dataflow.TaintTracking`) as well.
-  - This release includes preliminary support for interprocedural flow through
-    fields (non-static data members). In some cases, data stored in a field in
-    one function can now flow to a read of the same field in a different
-    function.
-  - The possibility of specifying barrier edges using
-    `isBarrierEdge`/`isSanitizerEdge` in data-flow and taint-tracking
-    configurations has been replaced with the option of specifying in- and
-    out-barriers on nodes by overriding `isBarrierIn`/`isSanitizerIn` and
-    `isBarrierOut`/`isSanitizerOut`. This should be simpler to use effectively,
-    as it does not require knowledge about the actual edges used internally by
-    the library.
-  - The library now models data flow through `std::swap`.
-  - Recursion through the `DataFlow` library is now always a compile error. Such recursion has been deprecated since release 1.16 in March 2018. If one `DataFlow::Configuration` needs to depend on the results of another, switch one of them to use one of the `DataFlow2` through `DataFlow4` libraries.
-- In the `semmle.code.cpp.dataflow.TaintTracking` library, the second copy of `Configuration` has been renamed from `TaintTracking::Configuration2` to `TaintTracking2::Configuration`, and the old name is now deprecated. Import `semmle.code.cpp.dataflow.TaintTracking2` to access the new name.
-- The `semmle.code.cpp.security.TaintTracking` library now considers a pointer difference calculation as blocking taint flow.
-- The predicate `Variable.getAnAssignedValue()` now reports assignments to fields resulting from aggregate initialization (` = {...}`).
-- The predicate `TypeMention.toString()` has been simplified to always return the string "`type mention`".  This may improve performance when using `Element.toString()` or its descendants.
-- Fixed the `LocalScopeVariableReachability.qll` library's handling of loops where the entry condition is always true on first entry, and where there is more than one control flow path through the loop condition. This change increases the accuracy of the `LocalScopeVariableReachability.qll` library and queries that depend on it.
-- There is a new `Variable.isThreadLocal()` predicate. It can be used to tell whether a variable is `thread_local`.
-- C/C++ code examples have been added to QLDoc comments on many more classes in the QL libraries.