From 9c792902c7eab3ae5451092d0209bdc28ea846e2 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Mon, 21 Nov 2022 16:25:26 +0100 Subject: [PATCH 1/6] Ruby: cache the entire extractor --- ruby/actions/create-extractor-pack/action.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ruby/actions/create-extractor-pack/action.yml b/ruby/actions/create-extractor-pack/action.yml index b0907eff834..667158c264c 100644 --- a/ruby/actions/create-extractor-pack/action.yml +++ b/ruby/actions/create-extractor-pack/action.yml @@ -3,7 +3,15 @@ description: Builds the Ruby CodeQL pack runs: using: composite steps: - - uses: actions/cache@v3 + - name: Cache entire extractor + id: cache-extractor + uses: actions/cache@v3 + with: + path: ruby/extractor-pack + key: ${{ runner.os }}-extractor-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}-${{ hashFiles('ruby/**/*.rs') }}-${{ hashFiles('ruby/codeql-extractor.yml', 'ruby/downgrades', 'ruby/tools', 'ruby/ql/lib/ruby.dbscheme', 'ruby/ql/lib/ruby.dbscheme.stats') }} + - name: Cache cargo + uses: actions/cache@v3 + if: steps.cache-extractor.outputs.cache-hit != 'true' with: path: | ~/.cargo/registry @@ -11,6 +19,7 @@ runs: ruby/target key: ${{ runner.os }}-ruby-qltest-cargo-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }} - name: Build Extractor + if: steps.cache-extractor.outputs.cache-hit != 'true' shell: bash run: scripts/create-extractor-pack.sh working-directory: ruby From b1db39020057ad96e2321ff08c6ca8da0e973613 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Mon, 21 Nov 2022 16:25:44 +0100 Subject: [PATCH 2/6] Ruby: use compilation cache in the ruby-build workflow --- .github/workflows/ruby-build.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ruby-build.yml b/.github/workflows/ruby-build.yml index 2c4ddb8a582..3ba334b94a2 100644 --- a/.github/workflows/ruby-build.yml +++ b/.github/workflows/ruby-build.yml @@ -93,13 +93,18 @@ jobs: - uses: actions/checkout@v3 - name: Fetch CodeQL uses: ./.github/actions/fetch-codeql + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: ruby-build - name: Build Query Pack run: | - codeql pack create ../shared/ssa --output target/packs - codeql pack create ../misc/suite-helpers --output target/packs - codeql pack create ../shared/regex --output target/packs - codeql pack create ql/lib --output target/packs - codeql pack create ql/src --output target/packs + codeql pack create ../shared/ssa --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" + codeql pack create ../misc/suite-helpers --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" + codeql pack create ../shared/regex --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" + codeql pack create ql/lib --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" + codeql pack create ql/src --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*) codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;) From 999e8ed0d08f442d0d51ddafdbd9b4e19985f21f Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Mon, 21 Nov 2022 16:30:15 +0100 Subject: [PATCH 3/6] Ruby: remove the path on branch pushes, for caching --- .github/workflows/ruby-build.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ruby-build.yml b/.github/workflows/ruby-build.yml index 3ba334b94a2..6895e3afeb1 100644 --- a/.github/workflows/ruby-build.yml +++ b/.github/workflows/ruby-build.yml @@ -2,11 +2,7 @@ name: "Ruby: Build" on: push: - paths: - - "ruby/**" - - .github/workflows/ruby-build.yml - - .github/actions/fetch-codeql/action.yml - - codeql-workspace.yml + # No path requirement on branch pushes, so the cache is more effective. branches: - main - "rc/*" From 3b7ce0680d51b977cf0d8df64ea124ea2499d2b4 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Mon, 21 Nov 2022 16:35:11 +0100 Subject: [PATCH 4/6] Ruby: build queries on an XL worker, and use all the threads --- .github/workflows/ruby-build.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ruby-build.yml b/.github/workflows/ruby-build.yml index 6895e3afeb1..cb9278f091e 100644 --- a/.github/workflows/ruby-build.yml +++ b/.github/workflows/ruby-build.yml @@ -82,9 +82,7 @@ jobs: ruby/target/release/ruby-extractor.exe retention-days: 1 compile-queries: - runs-on: ubuntu-latest - env: - CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI + runs-on: ubuntu-latest-xl steps: - uses: actions/checkout@v3 - name: Fetch CodeQL @@ -96,11 +94,11 @@ jobs: key: ruby-build - name: Build Query Pack run: | - codeql pack create ../shared/ssa --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" - codeql pack create ../misc/suite-helpers --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" - codeql pack create ../shared/regex --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" - codeql pack create ql/lib --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" - codeql pack create ql/src --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" + codeql pack create -j0 ../shared/ssa --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" + codeql pack create -j0 ../misc/suite-helpers --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" + codeql pack create -j0 ../shared/regex --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" + codeql pack create -j0 ql/lib --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" + codeql pack create -j0 ql/src --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*) codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;) From 53ba22ab5c565770e1ac0c89362184f29f7ce3bf Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Mon, 21 Nov 2022 17:37:11 +0100 Subject: [PATCH 5/6] simplify pack creation Co-authored-by: Arthur Baars --- .github/workflows/ruby-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ruby-build.yml b/.github/workflows/ruby-build.yml index cb9278f091e..abca7bd8913 100644 --- a/.github/workflows/ruby-build.yml +++ b/.github/workflows/ruby-build.yml @@ -94,10 +94,10 @@ jobs: key: ruby-build - name: Build Query Pack run: | - codeql pack create -j0 ../shared/ssa --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" - codeql pack create -j0 ../misc/suite-helpers --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" - codeql pack create -j0 ../shared/regex --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" - codeql pack create -j0 ql/lib --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" + codeql pack create ../shared/ssa --output target/packs + codeql pack create ../misc/suite-helpers --output target/packs + codeql pack create ../shared/regex --output target/packs + codeql pack create ql/lib --output target/packs codeql pack create -j0 ql/src --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*) codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src From 76ceb498417dd15ba181c81f32c9c1371be1be1b Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Mon, 21 Nov 2022 17:52:06 +0100 Subject: [PATCH 6/6] re-introduce the paths requirements in the ruby workflows --- .github/workflows/ruby-build.yml | 6 +++++- .github/workflows/ruby-qltest.yml | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ruby-build.yml b/.github/workflows/ruby-build.yml index abca7bd8913..784a7db3dc9 100644 --- a/.github/workflows/ruby-build.yml +++ b/.github/workflows/ruby-build.yml @@ -2,7 +2,11 @@ name: "Ruby: Build" on: push: - # No path requirement on branch pushes, so the cache is more effective. + paths: + - "ruby/**" + - .github/workflows/ruby-build.yml + - .github/actions/fetch-codeql/action.yml + - codeql-workspace.yml branches: - main - "rc/*" diff --git a/.github/workflows/ruby-qltest.yml b/.github/workflows/ruby-qltest.yml index abb6d2bf0eb..d8af552c8c6 100644 --- a/.github/workflows/ruby-qltest.yml +++ b/.github/workflows/ruby-qltest.yml @@ -2,7 +2,11 @@ name: "Ruby: Run QL Tests" on: push: - # no path requirement on branch pushes, so the cache is more effective. + paths: + - "ruby/**" + - .github/workflows/ruby-build.yml + - .github/actions/fetch-codeql/action.yml + - codeql-workspace.yml branches: - main - "rc/*"