From af58329931e4b8f1ae5c4ead2bb9722db5b4ba91 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Mon, 21 Nov 2022 14:30:29 +0100 Subject: [PATCH 1/6] split saving the compilation cache into a shared workflow --- .../cache-query-compilation/action.yml | 60 +++++++++++++++++++ .github/workflows/compile-queries.yml | 46 +++----------- 2 files changed, 67 insertions(+), 39 deletions(-) create mode 100644 .github/actions/cache-query-compilation/action.yml diff --git a/.github/actions/cache-query-compilation/action.yml b/.github/actions/cache-query-compilation/action.yml new file mode 100644 index 00000000000..55b1eb10ffa --- /dev/null +++ b/.github/actions/cache-query-compilation/action.yml @@ -0,0 +1,60 @@ +name: Cache query compilation +description: Caches CodeQL compilation caches - should be run both on PRs and pushes to main. + +inputs: + key: + description: 'The cache key to use - should be unique to the workflow' + required: true + +outputs: + cache-dir: + description: "The directory where the cache was stored" + value: ${{ steps.fill-compilation-dir.outputs.compdir }} + +runs: + using: composite + steps: + # Cache the query compilation caches. + # calculate the merge-base with main, in a way that works both on PRs and pushes to main. + - name: Calculate merge-base + shell: bash + if: ${{ github.event_name == 'pull_request' }} + env: + BASE_BRANCH: ${{ github.base_ref }} + run: | + MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ") + echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV + - name: Read CodeQL query compilation - PR + if: ${{ github.event_name == 'pull_request' }} + uses: actions/cache@v3 + with: + path: '**/.cache' + key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here. + restore-keys: | + codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge-base }} + codeql-compile-${{ inputs.key }}-${{ github.base_ref }}- + codeql-compile-${{ inputs.key }}-main- + - name: Fill CodeQL query compilation cache - main + if: ${{ github.event_name != 'pull_request' }} + uses: actions/cache@v3 + with: + path: '**/.cache' + key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main + restore-keys: | # restore from another random commit, to speed up compilation. + codeql-compile-${{ inputs.key }}-${{ github.ref_name }}- + codeql-compile-${{ inputs.key }}-main- + - name: Fill compilation cache directory + id: fill-compilation-dir + shell: bash + run: | + # Move all the existing cache into another folder, so we only preserve the cache for the current queries. + mkdir -p ${COMBINED_CACHE_DIR} + rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty. + # copy the contents of the .cache folders into the combined cache folder. + cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files + # clean up the .cache folders + rm -rf **/.cache/* + + echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT + env: + COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir \ No newline at end of file diff --git a/.github/workflows/compile-queries.yml b/.github/workflows/compile-queries.yml index d053b1cb7c1..ee6d48c45ea 100644 --- a/.github/workflows/compile-queries.yml +++ b/.github/workflows/compile-queries.yml @@ -14,58 +14,26 @@ jobs: steps: - uses: actions/checkout@v3 - # calculate the merge-base with main, in a way that works both on PRs and pushes to main. - - name: Calculate merge-base - if: ${{ github.event_name == 'pull_request' }} - env: - BASE_BRANCH: ${{ github.base_ref }} - run: | - MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ") - echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV - - name: Read CodeQL query compilation - PR - if: ${{ github.event_name == 'pull_request' }} - uses: actions/cache@v3 - with: - path: '**/.cache' - key: codeql-compile-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here. - restore-keys: | - codeql-compile-${{ github.base_ref }}-${{ env.merge-base }} - codeql-compile-${{ github.base_ref }}- - codeql-compile-main- - - name: Fill CodeQL query compilation cache - main - if: ${{ github.event_name != 'pull_request' }} - uses: actions/cache@v3 - with: - path: '**/.cache' - key: codeql-compile-${{ github.ref_name }}-${{ github.sha }} # just fill on main - restore-keys: | # restore from another random commit, to speed up compilation. - codeql-compile-${{ github.ref_name }}- - codeql-compile-main- - name: Setup CodeQL uses: ./.github/actions/fetch-codeql with: channel: 'release' + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: all-queries - name: check formatting run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only - name: compile queries - check-only # run with --check-only if running in a PR (github.sha != main) if : ${{ github.event_name == 'pull_request' }} shell: bash - run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only + run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" - name: compile queries - full # do full compile if running on main - this populates the cache if : ${{ github.event_name != 'pull_request' }} shell: bash - run: | - # Move all the existing cache into another folder, so we only preserve the cache for the current queries. - mkdir -p ${COMBINED_CACHE_DIR} - rm -f */ql/{src,examples}/.cache/{lock,size} # -f to avoid errors if the cache is empty. - # copy the contents of the .cache folders into the combined cache folder. - cp -r */ql/{src,examples}/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files - # clean up the .cache folders - rm -rf */ql/{src,examples}/.cache/* - - # compile the queries - codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache ${COMBINED_CACHE_DIR} + run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" env: COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir \ No newline at end of file From 24a973e545edd1fc4d5874713f7be05f0c205a5a Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Mon, 21 Nov 2022 14:31:24 +0100 Subject: [PATCH 2/6] run ruby qltest on a single XL worker --- .github/workflows/ruby-qltest.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ruby-qltest.yml b/.github/workflows/ruby-qltest.yml index 76727f626f0..dafeee4157c 100644 --- a/.github/workflows/ruby-qltest.yml +++ b/.github/workflows/ruby-qltest.yml @@ -48,17 +48,15 @@ jobs: xargs codeql execute upgrades testdb diff -q testdb/ruby.dbscheme downgrades/initial/ruby.dbscheme qltest: - runs-on: ubuntu-latest + runs-on: ubuntu-latest-xl strategy: fail-fast: false - matrix: - slice: ["1/2", "2/2"] steps: - uses: actions/checkout@v3 - uses: ./.github/actions/fetch-codeql - uses: ./ruby/actions/create-extractor-pack - name: Run QL tests run: | - codeql test run --threads=0 --ram 5000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test + codeql test run --threads=0 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test env: GITHUB_TOKEN: ${{ github.token }} From 4af8d5769a4efc19c548d8bf9a63fe582baf2c11 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Mon, 21 Nov 2022 14:32:18 +0100 Subject: [PATCH 3/6] use compilation cache in ruby qltest --- .github/workflows/ruby-qltest.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ruby-qltest.yml b/.github/workflows/ruby-qltest.yml index dafeee4157c..893e123ece5 100644 --- a/.github/workflows/ruby-qltest.yml +++ b/.github/workflows/ruby-qltest.yml @@ -55,8 +55,13 @@ jobs: - uses: actions/checkout@v3 - uses: ./.github/actions/fetch-codeql - uses: ./ruby/actions/create-extractor-pack + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: ruby-qltest - name: Run QL tests run: | - codeql test run --threads=0 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test + codeql test run --threads=0 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" env: GITHUB_TOKEN: ${{ github.token }} From a9c95a3230d4d4db5d353db9bf11110b03308155 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Mon, 21 Nov 2022 14:39:56 +0100 Subject: [PATCH 4/6] ruby: delete the path requirement when running qltest on branches --- .github/workflows/ruby-qltest.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ruby-qltest.yml b/.github/workflows/ruby-qltest.yml index 893e123ece5..a6cd41a55bd 100644 --- a/.github/workflows/ruby-qltest.yml +++ b/.github/workflows/ruby-qltest.yml @@ -2,11 +2,7 @@ name: "Ruby: Run QL Tests" on: push: - paths: - - "ruby/**" - - .github/workflows/ruby-qltest.yml - - .github/actions/fetch-codeql/action.yml - - codeql-workspace.yml + # no path requirement on branch pushes, so the cache is more effective. branches: - main - "rc/*" From f50778ae2646ff72d48a59abd59f77f9bd575346 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Mon, 21 Nov 2022 15:02:14 +0100 Subject: [PATCH 5/6] Ruby: set ram usage to 52GB in the qltest workflow --- .github/workflows/ruby-qltest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ruby-qltest.yml b/.github/workflows/ruby-qltest.yml index a6cd41a55bd..abb6d2bf0eb 100644 --- a/.github/workflows/ruby-qltest.yml +++ b/.github/workflows/ruby-qltest.yml @@ -58,6 +58,6 @@ jobs: key: ruby-qltest - name: Run QL tests run: | - codeql test run --threads=0 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" + codeql test run --threads=0 --ram 52000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" env: GITHUB_TOKEN: ${{ github.token }} From 57a7f8948596bef0c8021975ceba582b9a96fe16 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Mon, 21 Nov 2022 15:15:12 +0100 Subject: [PATCH 6/6] change merge-base to merge_base --- .github/actions/cache-query-compilation/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/cache-query-compilation/action.yml b/.github/actions/cache-query-compilation/action.yml index 55b1eb10ffa..c071aa204d9 100644 --- a/.github/actions/cache-query-compilation/action.yml +++ b/.github/actions/cache-query-compilation/action.yml @@ -23,7 +23,7 @@ runs: BASE_BRANCH: ${{ github.base_ref }} run: | MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ") - echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV + echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV - name: Read CodeQL query compilation - PR if: ${{ github.event_name == 'pull_request' }} uses: actions/cache@v3 @@ -31,7 +31,7 @@ runs: path: '**/.cache' key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here. restore-keys: | - codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge-base }} + codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }} codeql-compile-${{ inputs.key }}-${{ github.base_ref }}- codeql-compile-${{ inputs.key }}-main- - name: Fill CodeQL query compilation cache - main