diff --git a/.github/actions/cache-query-compilation/action.yml b/.github/actions/cache-query-compilation/action.yml new file mode 100644 index 00000000000..c071aa204d9 --- /dev/null +++ b/.github/actions/cache-query-compilation/action.yml @@ -0,0 +1,60 @@ +name: Cache query compilation +description: Caches CodeQL compilation caches - should be run both on PRs and pushes to main. + +inputs: + key: + description: 'The cache key to use - should be unique to the workflow' + required: true + +outputs: + cache-dir: + description: "The directory where the cache was stored" + value: ${{ steps.fill-compilation-dir.outputs.compdir }} + +runs: + using: composite + steps: + # Cache the query compilation caches. + # calculate the merge-base with main, in a way that works both on PRs and pushes to main. + - name: Calculate merge-base + shell: bash + if: ${{ github.event_name == 'pull_request' }} + env: + BASE_BRANCH: ${{ github.base_ref }} + run: | + MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ") + echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV + - name: Read CodeQL query compilation - PR + if: ${{ github.event_name == 'pull_request' }} + uses: actions/cache@v3 + with: + path: '**/.cache' + key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here. + restore-keys: | + codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }} + codeql-compile-${{ inputs.key }}-${{ github.base_ref }}- + codeql-compile-${{ inputs.key }}-main- + - name: Fill CodeQL query compilation cache - main + if: ${{ github.event_name != 'pull_request' }} + uses: actions/cache@v3 + with: + path: '**/.cache' + key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main + restore-keys: | # restore from another random commit, to speed up compilation. + codeql-compile-${{ inputs.key }}-${{ github.ref_name }}- + codeql-compile-${{ inputs.key }}-main- + - name: Fill compilation cache directory + id: fill-compilation-dir + shell: bash + run: | + # Move all the existing cache into another folder, so we only preserve the cache for the current queries. + mkdir -p ${COMBINED_CACHE_DIR} + rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty. + # copy the contents of the .cache folders into the combined cache folder. + cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files + # clean up the .cache folders + rm -rf **/.cache/* + + echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT + env: + COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir \ No newline at end of file diff --git a/.github/workflows/compile-queries.yml b/.github/workflows/compile-queries.yml index d053b1cb7c1..ee6d48c45ea 100644 --- a/.github/workflows/compile-queries.yml +++ b/.github/workflows/compile-queries.yml @@ -14,58 +14,26 @@ jobs: steps: - uses: actions/checkout@v3 - # calculate the merge-base with main, in a way that works both on PRs and pushes to main. - - name: Calculate merge-base - if: ${{ github.event_name == 'pull_request' }} - env: - BASE_BRANCH: ${{ github.base_ref }} - run: | - MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ") - echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV - - name: Read CodeQL query compilation - PR - if: ${{ github.event_name == 'pull_request' }} - uses: actions/cache@v3 - with: - path: '**/.cache' - key: codeql-compile-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here. - restore-keys: | - codeql-compile-${{ github.base_ref }}-${{ env.merge-base }} - codeql-compile-${{ github.base_ref }}- - codeql-compile-main- - - name: Fill CodeQL query compilation cache - main - if: ${{ github.event_name != 'pull_request' }} - uses: actions/cache@v3 - with: - path: '**/.cache' - key: codeql-compile-${{ github.ref_name }}-${{ github.sha }} # just fill on main - restore-keys: | # restore from another random commit, to speed up compilation. - codeql-compile-${{ github.ref_name }}- - codeql-compile-main- - name: Setup CodeQL uses: ./.github/actions/fetch-codeql with: channel: 'release' + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: all-queries - name: check formatting run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only - name: compile queries - check-only # run with --check-only if running in a PR (github.sha != main) if : ${{ github.event_name == 'pull_request' }} shell: bash - run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only + run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" - name: compile queries - full # do full compile if running on main - this populates the cache if : ${{ github.event_name != 'pull_request' }} shell: bash - run: | - # Move all the existing cache into another folder, so we only preserve the cache for the current queries. - mkdir -p ${COMBINED_CACHE_DIR} - rm -f */ql/{src,examples}/.cache/{lock,size} # -f to avoid errors if the cache is empty. - # copy the contents of the .cache folders into the combined cache folder. - cp -r */ql/{src,examples}/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files - # clean up the .cache folders - rm -rf */ql/{src,examples}/.cache/* - - # compile the queries - codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache ${COMBINED_CACHE_DIR} + run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" env: COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir \ No newline at end of file diff --git a/.github/workflows/ruby-qltest.yml b/.github/workflows/ruby-qltest.yml index 76727f626f0..abb6d2bf0eb 100644 --- a/.github/workflows/ruby-qltest.yml +++ b/.github/workflows/ruby-qltest.yml @@ -2,11 +2,7 @@ name: "Ruby: Run QL Tests" on: push: - paths: - - "ruby/**" - - .github/workflows/ruby-qltest.yml - - .github/actions/fetch-codeql/action.yml - - codeql-workspace.yml + # no path requirement on branch pushes, so the cache is more effective. branches: - main - "rc/*" @@ -48,17 +44,20 @@ jobs: xargs codeql execute upgrades testdb diff -q testdb/ruby.dbscheme downgrades/initial/ruby.dbscheme qltest: - runs-on: ubuntu-latest + runs-on: ubuntu-latest-xl strategy: fail-fast: false - matrix: - slice: ["1/2", "2/2"] steps: - uses: actions/checkout@v3 - uses: ./.github/actions/fetch-codeql - uses: ./ruby/actions/create-extractor-pack + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: ruby-qltest - name: Run QL tests run: | - codeql test run --threads=0 --ram 5000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test + codeql test run --threads=0 --ram 52000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" env: GITHUB_TOKEN: ${{ github.token }}