Merge pull request #361 from github/prepare-merge

Prepare merge into github/codeql

.devcontainer/Dockerfile

@@ -1,15 +0,0 @@
-# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.162.0/containers/rust/.devcontainer/base.Dockerfile
-
-FROM mcr.microsoft.com/vscode/devcontainers/rust:0-1
-
-RUN apt-key --keyring /usr/share/keyrings/githubcli-archive-keyring.gpg adv \
-    --keyserver keyserver.ubuntu.com --recv-key C99B11DEB97541F0 && \
-    echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages $(lsb_release -cs) main" \
-    |  tee /etc/apt/sources.list.d/github-cli2.list > /dev/null
-
-
-RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends gh 
-
-COPY post_create.sh /bin/post_create.sh
-COPY post_attach.sh /bin/post_attach.sh

.devcontainer/devcontainer.json

@@ -1,39 +0,0 @@
-// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
-// https://github.com/microsoft/vscode-dev-containers/tree/v0.162.0/containers/rust
-{
-	"name": "Rust",
-	"build": {
-		"dockerfile": "Dockerfile"
-	},
-	"runArgs": [
-		"--cap-add=SYS_PTRACE",
-		"--security-opt",
-		"seccomp=unconfined"
-	],
-	// Set *default* container specific settings.json values on container create.
-	"settings": {
-		"terminal.integrated.shell.linux": "/bin/bash",
-		"lldb.executable": "/usr/bin/lldb",
-		// VS Code don't watch files under ./target
-		"files.watcherExclude": {
-			"**/target/**": true
-		}
-	},
-	// Add the IDs of extensions you want installed when the container is created.
-	"extensions": [
-		"rust-lang.rust",
-		"bungcip.better-toml",
-		"vadimcn.vscode-lldb",
-		"mutantdino.resourcemonitor",
-		"ms-azuretools.vscode-docker",
-		"github.vscode-codeql"
-	],
-	// Use 'forwardPorts' to make a list of ports inside the container available locally.
-	// "forwardPorts": [],
-	// Use 'postCreateCommand' to run commands after the container is created.
-	// "postCreateCommand": "rustc --version",
-	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
-	"remoteUser": "vscode",
-	"postCreateCommand": [ "/bin/post_create.sh" ],
-	"postAttachCommand": [ "flock", "-E", "0", "-n", "/var/lock/post_attach.lock", "/bin/post_attach.sh" ]
-}

.devcontainer/post_attach.sh

@@ -1,37 +0,0 @@
-#! /bin/bash
-set -xe
-
-echo "Check installed CodeQL version"
-CURRENT_CODEQL_BIN=$(readlink -e /usr/local/bin/codeql || echo "")
-LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
-
-BASE_DIR=/home/vscode/codeql-binaries
-mkdir -p "${BASE_DIR}"
-LATEST_CODEQL_DIR="${BASE_DIR}/codeql-${LATEST}"
-LATEST_CODEQL_BIN="${LATEST_CODEQL_DIR}/codeql/codeql"
-
-if [ "${CURRENT_CODEQL_BIN}" != "${LATEST_CODEQL_BIN}" ]; then
-  echo "Installing CodeQL ${LATEST}"
-  TMPDIR=$(mktemp -d -p "$(dirname ${LATEST_CODEQL_DIR})")
-  gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip -D "${TMPDIR}" "$LATEST" 
-  unzip -oq "${TMPDIR}/codeql-linux64.zip" -d "${TMPDIR}"
-  rm -f "${TMPDIR}/codeql-linux64.zip"
-  mv "${TMPDIR}" "${LATEST_CODEQL_DIR}"
-  test -x "${LATEST_CODEQL_BIN}" && sudo ln -sf "${LATEST_CODEQL_BIN}" /usr/local/bin/codeql
-  if [[ "${CURRENT_CODEQL_BIN}" =~ .*/codeql/codeql ]]; then
-    rm -rf "$(dirname $(dirname ${CURRENT_CODEQL_BIN}))"
-  fi
-fi
-
-echo "Build the Ruby extractor"
-
-# clone the git dependencies using "git clone" because cargo's builtin git support is rather slow
-REPO_DIR="${CARGO_HOME:-/home/vscode/.cargo}/git/db" 
-REPO_DIR_ERB="${REPO_DIR}/tree-sitter-embedded-template-4c796e3340c233b6"
-REPO_DIR_RUBY="${REPO_DIR}/tree-sitter-ruby-666a40ce046f8e7a"
-
-mkdir -p "${REPO_DIR}"
-test -e "${REPO_DIR_ERB}" || git clone -q --bare https://github.com/tree-sitter/tree-sitter-embedded-template "${REPO_DIR_ERB}"
-test -e "${REPO_DIR_RUBY}" || git clone -q --bare https://github.com/tree-sitter/tree-sitter-ruby.git "${REPO_DIR_RUBY}"
-
-scripts/create-extractor-pack.sh

.devcontainer/post_create.sh

@@ -1,4 +0,0 @@
-#! /bin/bash
-
-mkdir -p /home/vscode/.config/codeql
-echo '--search-path /workspaces/codeql-ruby' >> /home/vscode/.config/codeql/config

.github/actions/fetch-codeql/action.yml

@@ -9,5 +9,6 @@ runs:
         LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
         gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
         unzip -q codeql-linux64.zip
+        echo "${{ github.workspace }}/codeql" >> $GITHUB_PATH
       env:
         GITHUB_TOKEN: ${{ github.token }}

.github/dependabot.yml

@@ -1,18 +1,18 @@
 version: 2
 updates:
   - package-ecosystem: "cargo"
-    directory: "/node-types"
+    directory: "ruby/node-types"
     schedule:
       interval: "daily"
   - package-ecosystem: "cargo"
-    directory: "/generator"
+    directory: "ruby/generator"
     schedule:
       interval: "daily"
   - package-ecosystem: "cargo"
-    directory: "/extractor"
+    directory: "ruby/extractor"
     schedule:
       interval: "daily"
   - package-ecosystem: "cargo"
-    directory: "/autobuilder"
+    directory: "ruby/autobuilder"
     schedule:
       interval: "daily"

.github/workflows/qhelp-pr-preview.yml

@@ -6,7 +6,7 @@ on:
       - main
       - 'rc/*'
     paths:
-      - "**/*.qhelp"
+      - "ruby/**/*.qhelp"
 
 jobs:
   qhelp:
@@ -32,7 +32,7 @@ jobs:
           for path in ${{ steps.changes.outputs.qhelp_files }} ; do
             echo "<details> <summary>${path}</summary>"
             echo
-            codeql/codeql generate query-help --format=markdown ${path}
+            codeql generate query-help --format=markdown ${path}
             echo "</details>"
           done) | gh pr comment "${{ github.event.pull_request.number }}" -F -
         env:

.github/workflows/qltest.yml

@@ -1,42 +0,0 @@
-name: Run QL Tests
-
-on:
-  push:
-    branches:
-      - main
-      - 'rc/*'
-  pull_request:
-    branches:
-      - main
-      - 'rc/*'
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  qltest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: ./.github/actions/fetch-codeql
-      - uses: ./.github/actions/create-extractor-pack
-      - name: Run QL tests
-        run: |
-          codeql/codeql pack install ql/test
-          codeql/codeql test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --additional-packs "${HOME}/.codeql/packages/codeql/suite-helpers/0.0.1" --consistency-queries ql/consistency-queries ql/test
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-      - name: Check QL formatting
-        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql/codeql query format --check-only
-      - name: Check QL compilation
-        run: |
-          codeql/codeql pack install ql/src
-          codeql/codeql query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}" "ql/src" "ql/examples"
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-      - name: Check DB upgrade scripts
-        run: |
-          echo >empty.trap
-          codeql/codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
-          codeql/codeql dataset upgrade testdb --additional-packs ql/lib/upgrades
-          diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme

.github/workflows/ruby-build.yml

@@ -1,11 +1,15 @@
-name: Build / Release
+name: "Ruby: Build"
 
 on:
   push:
+    paths:
+      - 'ruby/**'
     branches:
       - main
       - 'rc/*'
   pull_request:
+    paths:
+      - 'ruby/**'
     branches:
       - main
       - 'rc/*'
@@ -18,6 +22,10 @@ on:
 env:
   CARGO_TERM_COLOR: always
 
+defaults:
+  run:
+    working-directory: ruby
+
 jobs:
   build:
     strategy:
@@ -39,7 +47,7 @@ jobs:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
-            target
+            ruby/target
           key: ${{ runner.os }}-rust-cargo-${{ hashFiles('**/Cargo.lock') }}
       - name: Check formatting
         run: cargo fmt --all -- --check
@@ -56,20 +64,20 @@ jobs:
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           name: ruby.dbscheme
-          path: ql/lib/ruby.dbscheme
+          path: ruby/ql/lib/ruby.dbscheme
       - uses: actions/upload-artifact@v2
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           name: TreeSitter.qll
-          path: ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+          path: ruby/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v2
         with:
           name: extractor-${{ matrix.os }}
           path: |
-            target/release/ruby-autobuilder
-            target/release/ruby-autobuilder.exe
-            target/release/ruby-extractor
-            target/release/ruby-extractor.exe
+            ruby/target/release/ruby-autobuilder
+            ruby/target/release/ruby-autobuilder.exe
+            ruby/target/release/ruby-extractor
+            ruby/target/release/ruby-extractor.exe
           retention-days: 1
   compile-queries:
     runs-on: ubuntu-latest
@@ -106,7 +114,7 @@ jobs:
         with:
           name: codeql-ruby-queries
           path: |
-            target/packs/*
+            ruby/target/packs/*
           retention-days: 1
 
   package:
@@ -117,19 +125,19 @@ jobs:
       - uses: actions/download-artifact@v2
         with:
           name: ruby.dbscheme
-          path: ruby
+          path: ruby/ruby
       - uses: actions/download-artifact@v2
         with:
           name: extractor-ubuntu-latest
-          path: linux64
+          path: ruby/linux64
       - uses: actions/download-artifact@v2
         with:
           name: extractor-windows-latest
-          path: win64
+          path: ruby/win64
       - uses: actions/download-artifact@v2
         with:
           name: extractor-macos-latest
-          path: osx64
+          path: ruby/osx64
       - run: |
           mkdir -p ruby
           cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
@@ -145,12 +153,12 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: codeql-ruby-pack
-          path: codeql-ruby.zip
+          path: ruby/codeql-ruby.zip
           retention-days: 1
       - uses: actions/download-artifact@v2
         with:
           name: codeql-ruby-queries
-          path: qlpacks
+          path: ruby/qlpacks
       - run: |
           echo '{
             "provide": [
@@ -162,18 +170,13 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: codeql-ruby-bundle
-          path: codeql-ruby-bundle.zip
+          path: ruby/codeql-ruby-bundle.zip
           retention-days: 1
-      - if: ${{ github.event.inputs.tag }}
-        run: |
-          gh release create --prerelease \
-             --title "CodeQL Ruby (${{ github.event.inputs.tag }})" \
-             --target "${{ github.sha }}" \
-             "${{ github.event.inputs.tag }}" \
-             codeql-ruby-bundle.zip
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   test:
+    defaults:
+      run:
+        working-directory: ${{ github.workspace }}
     strategy:
       fail-fast: false
       matrix:

.github/workflows/ruby-dataset-measure.yml

@@ -1,4 +1,4 @@
-name: Collect database stats
+name: "Ruby: Collect database stats"
 
 on:
   push:
@@ -6,13 +6,13 @@ on:
       - main
       - 'rc/*'
     paths:
-      - ql/lib/ruby.dbscheme
+      - ruby/ql/lib/ruby.dbscheme
   pull_request:
     branches:
       - main
       - 'rc/*'
     paths:
-      - ql/lib/ruby.dbscheme
+      - ruby/ql/lib/ruby.dbscheme
   workflow_dispatch:
 
 jobs:
@@ -29,7 +29,7 @@ jobs:
 
       - uses: ./.github/actions/fetch-codeql
 
-      - uses: ./.github/actions/create-extractor-pack
+      - uses: ./ruby/actions/create-extractor-pack
 
       - name: Checkout ${{ matrix.repo }}
         uses: actions/checkout@v2
@@ -38,15 +38,15 @@ jobs:
           path: ${{ github.workspace }}/repo
       - name: Create database
         run: |
-          codeql/codeql database create \
-            --search-path "${{ github.workspace }}" \
+          codeql database create \
+            --search-path "${{ github.workspace }}/ruby" \
             --threads 4 \
             --language ruby --source-root "${{ github.workspace }}/repo" \
             "${{ runner.temp }}/database"
       - name: Measure database
         run: |
           mkdir -p "stats/${{ matrix.repo }}"
-          codeql/codeql dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ruby"
+          codeql dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ruby"
       - uses: actions/upload-artifact@v2
         with:
           name: measurements
@@ -64,8 +64,8 @@ jobs:
           path: stats
       - run: |
           python -m pip install --user lxml
-          find stats -name 'stats.xml' | sort | xargs python scripts/merge_stats.py --output ql/lib/ruby.dbscheme.stats --normalise ruby_tokeninfo
+          find stats -name 'stats.xml' | sort | xargs python ruby/scripts/merge_stats.py --output ruby/ql/lib/ruby.dbscheme.stats --normalise ruby_tokeninfo
       - uses: actions/upload-artifact@v2
         with:
           name: ruby.dbscheme.stats
-          path: ql/lib/ruby.dbscheme.stats
+          path: ruby/ql/lib/ruby.dbscheme.stats

.github/workflows/ruby-qltest.yml

@@ -0,0 +1,50 @@
+name: "Ruby: Run QL Tests"
+
+on:
+  push:
+    paths:
+      - 'ruby/**'
+    branches:
+      - main
+      - 'rc/*'
+  pull_request:
+    paths:
+      - 'ruby/**'
+    branches:
+      - main
+      - 'rc/*'
+
+env:
+  CARGO_TERM_COLOR: always
+
+defaults:
+  run:
+    working-directory: ruby
+
+jobs:
+  qltest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: ./.github/actions/fetch-codeql
+      - uses: ./ruby/actions/create-extractor-pack
+      - name: Run QL tests
+        run: |
+          codeql test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ruby" --additional-packs "${{ github.workspace }}/ruby/codeql"  --consistency-queries ql/consistency-queries ql/test
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - name: Check QL formatting
+        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
+      - name: Check QL compilation
+        run: |
+          codeql query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}/ruby" --additional-packs "${{ github.workspace }}/ruby/codeql" "ql/src" "ql/examples"
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - name: Check DB upgrade scripts
+        run: |
+          echo >empty.trap
+          codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
+          codeql dataset upgrade testdb --additional-packs ql/lib/upgrades
+          diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme

.github/workflows/sync-files.yml

@@ -18,4 +18,4 @@ jobs:
         with:
           submodules: true
       - name: Check synchronized files
-        run: scripts/sync-identical-files.py
+        run: ruby/scripts/sync-identical-files.py

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "codeql"]
-	path = codeql
+	path = ruby/codeql
 	url = https://github.com/github/codeql.git

CODE_OF_CONDUCT.md

@@ -1,76 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to make participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, sex characteristics, gender identity and expression,
-level of experience, education, socio-economic status, nationality, personal
-appearance, race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or
-  advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies within all project spaces, and it also applies when
-an individual is representing the project or its community in public spaces.
-Examples of representing a project or community include using an official
-project e-mail address, posting via an official social media account, or acting
-as an appointed representative at an online or offline event. Representation of
-a project may be further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at opensource@github.com. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
-
-[homepage]: https://www.contributor-covenant.org
-
-For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq

LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2020-2021 GitHub
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

ruby/actions/create-extractor-pack/action.yml

@@ -8,8 +8,9 @@ runs:
         path: |
           ~/.cargo/registry
           ~/.cargo/git
-          target
-        key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
+          ruby/target
+        key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ruby/**/Cargo.lock') }}
     - name: Build Extractor
       shell: bash
-      run: env "PATH=$PATH:${{ github.workspace }}/codeql" scripts/create-extractor-pack.sh
+      run: scripts/create-extractor-pack.sh
+      working-directory: ruby

ruby/change-notes/2021-10-14-codeql-ruby-beta.md

@@ -0,0 +1,2 @@
+codescanning
+* Open-sourcing the Ruby extractor and CodeQL queries as part of the [public beta](https://github.com/github/roadmap/issues/136) release.