mirror of
https://github.com/github/codeql.git
synced 2026-06-10 23:41:09 +02:00
Merge branch 'main' into copilot/automate-go-version-updates-again
This commit is contained in:
Owen Mansel-Chan
78
.github/workflows/compile-queries.yml
vendored
78
.github/workflows/compile-queries.yml
vendored
@@ -1,78 +0,0 @@
|
||||
name: "Compile all queries using the latest stable CodeQL CLI"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: # makes sure the cache gets populated - running on the branches people tend to merge into.
|
||||
- main
|
||||
- "rc/*"
|
||||
- "codeql-cli-*"
|
||||
pull_request:
|
||||
paths:
|
||||
- '**.ql'
|
||||
- '**.qll'
|
||||
- '**/qlpack.yml'
|
||||
- '**.dbscheme'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
if: github.repository_owner == 'github'
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
languages: ${{ steps.detect.outputs.languages }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- name: Detect changed languages
|
||||
id: detect
|
||||
run: |
|
||||
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
|
||||
# For PRs, detect which languages have changes
|
||||
changed_files=$(gh pr view ${{ github.event.pull_request.number }} --json files --jq '.files.[].path')
|
||||
languages=()
|
||||
for lang in actions cpp csharp go java javascript python ql ruby rust swift; do
|
||||
if echo "$changed_files" | grep -qE "^($lang/|shared/)" ; then
|
||||
languages+=("$lang")
|
||||
fi
|
||||
done
|
||||
echo "languages=$(jq -c -n '$ARGS.positional' --args "${languages[@]}")" >> $GITHUB_OUTPUT
|
||||
else
|
||||
# For pushes to main/rc branches, run all languages
|
||||
echo 'languages=["actions","cpp","csharp","go","java","javascript","python","ql","ruby","rust","swift"]' >> $GITHUB_OUTPUT
|
||||
fi
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
compile-queries:
|
||||
needs: detect-changes
|
||||
if: github.repository_owner == 'github' && needs.detect-changes.outputs.languages != '[]'
|
||||
runs-on: ubuntu-latest-xl
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: ${{ fromJson(needs.detect-changes.outputs.languages) }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- name: Setup CodeQL
|
||||
uses: ./.github/actions/fetch-codeql
|
||||
with:
|
||||
channel: 'release'
|
||||
- name: Cache compilation cache
|
||||
id: query-cache
|
||||
uses: ./.github/actions/cache-query-compilation
|
||||
with:
|
||||
key: ${{ matrix.language }}-queries
|
||||
- name: check formatting
|
||||
run: find shared ${{ matrix.language }}/ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 -n 3000 -P 10 codeql query format -q --check-only
|
||||
- name: compile queries - check-only
|
||||
# run with --check-only if running in a PR (github.sha != main)
|
||||
if : ${{ github.event_name == 'pull_request' }}
|
||||
shell: bash
|
||||
run: codeql query compile -q -j0 ${{ matrix.language }}/ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" --compilation-cache-size=500 --ram=56000
|
||||
- name: compile queries - full
|
||||
# do full compile if running on main - this populates the cache
|
||||
if : ${{ github.event_name != 'pull_request' }}
|
||||
shell: bash
|
||||
run: codeql query compile -q -j0 ${{ matrix.language }}/ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" --compilation-cache-size=500 --ram=56000
|
||||
236
.github/workflows/ruby-build.yml
vendored
236
.github/workflows/ruby-build.yml
vendored
@@ -1,236 +0,0 @@
|
||||
name: "Ruby: Build"
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "ruby/**"
|
||||
- .github/workflows/ruby-build.yml
|
||||
- .github/actions/fetch-codeql/action.yml
|
||||
- codeql-workspace.yml
|
||||
- "shared/tree-sitter-extractor/**"
|
||||
branches:
|
||||
- main
|
||||
- "rc/*"
|
||||
pull_request:
|
||||
paths:
|
||||
- "ruby/**"
|
||||
- .github/workflows/ruby-build.yml
|
||||
- .github/actions/fetch-codeql/action.yml
|
||||
- codeql-workspace.yml
|
||||
- "shared/tree-sitter-extractor/**"
|
||||
branches:
|
||||
- main
|
||||
- "rc/*"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Version tag to create"
|
||||
required: false
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ruby
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- name: Install GNU tar
|
||||
if: runner.os == 'macOS'
|
||||
run: |
|
||||
brew install gnu-tar
|
||||
echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
|
||||
- name: Prepare Windows
|
||||
if: runner.os == 'Windows'
|
||||
shell: powershell
|
||||
run: |
|
||||
git config --global core.longpaths true
|
||||
- uses: ./.github/actions/os-version
|
||||
id: os_version
|
||||
- name: Cache entire extractor
|
||||
uses: actions/cache@v3
|
||||
id: cache-extractor
|
||||
with:
|
||||
path: |
|
||||
target/release/codeql-extractor-ruby
|
||||
target/release/codeql-extractor-ruby.exe
|
||||
ruby/extractor/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
|
||||
key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ruby/extractor/**/*.rs') }}
|
||||
- uses: actions/cache@v3
|
||||
if: steps.cache-extractor.outputs.cache-hit != 'true'
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-rust-cargo-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/**/Cargo.lock') }}
|
||||
- name: Check formatting
|
||||
if: steps.cache-extractor.outputs.cache-hit != 'true'
|
||||
run: cd extractor && cargo fmt -- --check
|
||||
- name: Build
|
||||
if: steps.cache-extractor.outputs.cache-hit != 'true'
|
||||
run: cd extractor && cargo build --verbose
|
||||
- name: Run tests
|
||||
if: steps.cache-extractor.outputs.cache-hit != 'true'
|
||||
run: cd extractor && cargo test --verbose
|
||||
- name: Release build
|
||||
if: steps.cache-extractor.outputs.cache-hit != 'true'
|
||||
run: cd extractor && cargo build --release
|
||||
- name: Generate dbscheme
|
||||
if: ${{ matrix.os == 'ubuntu-latest' && steps.cache-extractor.outputs.cache-hit != 'true'}}
|
||||
run: ../target/release/codeql-extractor-ruby generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ matrix.os == 'ubuntu-latest' }}
|
||||
with:
|
||||
name: ruby.dbscheme
|
||||
path: ruby/ql/lib/ruby.dbscheme
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ matrix.os == 'ubuntu-latest' }}
|
||||
with:
|
||||
name: TreeSitter.qll
|
||||
path: ruby/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: extractor-${{ matrix.os }}
|
||||
path: |
|
||||
target/release/codeql-extractor-ruby
|
||||
target/release/codeql-extractor-ruby.exe
|
||||
retention-days: 1
|
||||
compile-queries:
|
||||
if: github.repository_owner == 'github'
|
||||
runs-on: ubuntu-latest-xl
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- name: Fetch CodeQL
|
||||
uses: ./.github/actions/fetch-codeql
|
||||
- name: Cache compilation cache
|
||||
id: query-cache
|
||||
uses: ./.github/actions/cache-query-compilation
|
||||
with:
|
||||
key: ruby-build
|
||||
- name: Build Query Pack
|
||||
run: |
|
||||
PACKS=${{ runner.temp }}/query-packs
|
||||
rm -rf $PACKS
|
||||
codeql pack create ../misc/suite-helpers --output "$PACKS"
|
||||
codeql pack create ../shared/regex --output "$PACKS"
|
||||
codeql pack create ../shared/ssa --output "$PACKS"
|
||||
codeql pack create ../shared/tutorial --output "$PACKS"
|
||||
codeql pack create ql/lib --output "$PACKS"
|
||||
codeql pack create -j0 ql/src --output "$PACKS" --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
|
||||
PACK_FOLDER=$(readlink -f "$PACKS"/codeql/ruby-queries/*)
|
||||
codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
|
||||
(cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: codeql-ruby-queries
|
||||
path: |
|
||||
${{ runner.temp }}/query-packs/*
|
||||
retention-days: 1
|
||||
include-hidden-files: true
|
||||
|
||||
package:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build, compile-queries]
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ruby.dbscheme
|
||||
path: ruby/ruby
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: extractor-ubuntu-latest
|
||||
path: ruby/linux64
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: extractor-windows-latest
|
||||
path: ruby/win64
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: extractor-macos-latest
|
||||
path: ruby/osx64
|
||||
- run: |
|
||||
mkdir -p ruby
|
||||
cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
|
||||
mkdir -p ruby/tools/{linux64,osx64,win64}
|
||||
cp linux64/codeql-extractor-ruby ruby/tools/linux64/extractor
|
||||
cp osx64/codeql-extractor-ruby ruby/tools/osx64/extractor
|
||||
cp win64/codeql-extractor-ruby.exe ruby/tools/win64/extractor.exe
|
||||
chmod +x ruby/tools/{linux64,osx64}/extractor
|
||||
zip -rq codeql-ruby.zip ruby
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: codeql-ruby-pack
|
||||
path: ruby/codeql-ruby.zip
|
||||
retention-days: 1
|
||||
include-hidden-files: true
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: codeql-ruby-queries
|
||||
path: ruby/qlpacks
|
||||
- run: |
|
||||
echo '{
|
||||
"provide": [
|
||||
"ruby/codeql-extractor.yml",
|
||||
"qlpacks/*/*/*/qlpack.yml"
|
||||
]
|
||||
}' > .codeqlmanifest.json
|
||||
zip -rq codeql-ruby-bundle.zip .codeqlmanifest.json ruby qlpacks
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: codeql-ruby-bundle
|
||||
path: ruby/codeql-ruby-bundle.zip
|
||||
retention-days: 1
|
||||
include-hidden-files: true
|
||||
|
||||
test:
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ${{ github.workspace }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
needs: [package]
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- name: Fetch CodeQL
|
||||
uses: ./.github/actions/fetch-codeql
|
||||
|
||||
- name: Download Ruby bundle
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: codeql-ruby-bundle
|
||||
path: ${{ runner.temp }}
|
||||
- name: Unzip Ruby bundle
|
||||
shell: bash
|
||||
run: unzip -q -d "${{ runner.temp }}/ruby-bundle" "${{ runner.temp }}/codeql-ruby-bundle.zip"
|
||||
|
||||
- name: Run QL test
|
||||
shell: bash
|
||||
run: |
|
||||
codeql test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" ruby/ql/test/library-tests/ast/constants/
|
||||
- name: Create database
|
||||
shell: bash
|
||||
run: |
|
||||
codeql database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root ruby/ql/test/library-tests/ast/constants/ ../database
|
||||
- name: Analyze database
|
||||
shell: bash
|
||||
run: |
|
||||
codeql database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls
|
||||
75
.github/workflows/ruby-dataset-measure.yml
vendored
75
.github/workflows/ruby-dataset-measure.yml
vendored
@@ -1,75 +0,0 @@
|
||||
name: "Ruby: Collect database stats"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "rc/*"
|
||||
paths:
|
||||
- ruby/ql/lib/ruby.dbscheme
|
||||
- .github/workflows/ruby-dataset-measure.yml
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "rc/*"
|
||||
paths:
|
||||
- ruby/ql/lib/ruby.dbscheme
|
||||
- .github/workflows/ruby-dataset-measure.yml
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
measure:
|
||||
env:
|
||||
CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
repo: [rails/rails, discourse/discourse, spree/spree, ruby/ruby]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- uses: ./.github/actions/fetch-codeql
|
||||
|
||||
- uses: ./ruby/actions/create-extractor-pack
|
||||
|
||||
- name: Checkout ${{ matrix.repo }}
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
repository: ${{ matrix.repo }}
|
||||
path: ${{ github.workspace }}/repo
|
||||
- name: Create database
|
||||
run: |
|
||||
codeql database create \
|
||||
--search-path "${{ github.workspace }}" \
|
||||
--threads 4 \
|
||||
--language ruby --source-root "${{ github.workspace }}/repo" \
|
||||
"${{ runner.temp }}/database"
|
||||
- name: Measure database
|
||||
run: |
|
||||
mkdir -p "stats/${{ matrix.repo }}"
|
||||
codeql dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ruby"
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: measurements-${{ hashFiles('stats/**') }}
|
||||
path: stats
|
||||
retention-days: 1
|
||||
|
||||
merge:
|
||||
runs-on: ubuntu-latest
|
||||
needs: measure
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: stats
|
||||
- run: |
|
||||
python -m pip install --user lxml
|
||||
find stats -name 'stats.xml' | sort | xargs python ruby/scripts/merge_stats.py --output ruby/ql/lib/ruby.dbscheme.stats --normalise ruby_tokeninfo
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ruby.dbscheme.stats
|
||||
path: ruby/ql/lib/ruby.dbscheme.stats
|
||||
40
.github/workflows/ruby-qltest-rtjo.yml
vendored
40
.github/workflows/ruby-qltest-rtjo.yml
vendored
@@ -1,40 +0,0 @@
|
||||
name: "Ruby: Run RTJO Language Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- labeled
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ruby
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
qltest-rtjo:
|
||||
if: "github.repository_owner == 'github' && github.event.label.name == 'Run: RTJO Language Tests'"
|
||||
runs-on: ubuntu-latest-xl
|
||||
strategy:
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: ./.github/actions/fetch-codeql
|
||||
- uses: ./ruby/actions/create-extractor-pack
|
||||
- name: Cache compilation cache
|
||||
id: query-cache
|
||||
uses: ./.github/actions/cache-query-compilation
|
||||
with:
|
||||
key: ruby-qltest
|
||||
- name: Run QL tests
|
||||
run: |
|
||||
codeql test run --dynamic-join-order-mode=all --threads=0 --ram 50000 --search-path "${{ github.workspace }}" --check-databases --check-diff-informed --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
73
.github/workflows/ruby-qltest.yml
vendored
73
.github/workflows/ruby-qltest.yml
vendored
@@ -1,73 +0,0 @@
|
||||
name: "Ruby: Run QL Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "ruby/**"
|
||||
- "shared/**"
|
||||
- .github/workflows/ruby-build.yml
|
||||
- .github/actions/fetch-codeql/action.yml
|
||||
- codeql-workspace.yml
|
||||
branches:
|
||||
- main
|
||||
- "rc/*"
|
||||
pull_request:
|
||||
paths:
|
||||
- "ruby/**"
|
||||
- "shared/**"
|
||||
- .github/workflows/ruby-qltest.yml
|
||||
- .github/actions/fetch-codeql/action.yml
|
||||
- codeql-workspace.yml
|
||||
branches:
|
||||
- main
|
||||
- "rc/*"
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ruby
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
qlupgrade:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: ./.github/actions/fetch-codeql
|
||||
- name: Check DB upgrade scripts
|
||||
run: |
|
||||
echo >empty.trap
|
||||
codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
|
||||
codeql dataset upgrade testdb --additional-packs ql/lib
|
||||
diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme
|
||||
- name: Check DB downgrade scripts
|
||||
run: |
|
||||
echo >empty.trap
|
||||
rm -rf testdb; codeql dataset import -S ql/lib/ruby.dbscheme testdb empty.trap
|
||||
codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
|
||||
--dbscheme=ql/lib/ruby.dbscheme --target-dbscheme=downgrades/initial/ruby.dbscheme |
|
||||
xargs codeql execute upgrades testdb
|
||||
diff -q testdb/ruby.dbscheme downgrades/initial/ruby.dbscheme
|
||||
qltest:
|
||||
if: github.repository_owner == 'github'
|
||||
runs-on: ubuntu-latest-xl
|
||||
strategy:
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: ./.github/actions/fetch-codeql
|
||||
- uses: ./ruby/actions/create-extractor-pack
|
||||
- name: Cache compilation cache
|
||||
id: query-cache
|
||||
uses: ./.github/actions/cache-query-compilation
|
||||
with:
|
||||
key: ruby-qltest
|
||||
- name: Run QL tests
|
||||
run: |
|
||||
codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}" --check-databases --check-diff-informed --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
Reference in New Issue
Block a user