Merge pull request #11344 from erik-krogh/all-the-cache

Ruby: Use compilation cache for the qltest CI workflow

.github/actions/cache-query-compilation/action.yml

@@ -0,0 +1,60 @@
+name: Cache query compilation
+description: Caches CodeQL compilation caches - should be run both on PRs and pushes to main.
+
+inputs:
+  key:
+    description: 'The cache key to use - should be unique to the workflow'
+    required: true
+
+outputs:
+  cache-dir:
+    description: "The directory where the cache was stored"
+    value: ${{ steps.fill-compilation-dir.outputs.compdir }}
+
+runs:
+  using: composite
+  steps:
+    # Cache the query compilation caches.
+    # calculate the merge-base with main, in a way that works both on PRs and pushes to main. 
+    - name: Calculate merge-base
+      shell: bash
+      if: ${{ github.event_name == 'pull_request' }}
+      env:
+        BASE_BRANCH: ${{ github.base_ref }}
+      run: |
+        MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
+        echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV
+    - name: Read CodeQL query compilation - PR
+      if: ${{ github.event_name == 'pull_request' }}
+      uses: actions/cache@v3
+      with:
+        path: '**/.cache'
+        key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
+        restore-keys: |
+          codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
+          codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-
+          codeql-compile-${{ inputs.key }}-main-
+    - name: Fill CodeQL query compilation cache - main
+      if: ${{ github.event_name != 'pull_request' }}
+      uses: actions/cache@v3
+      with:
+        path: '**/.cache'
+        key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
+        restore-keys: | # restore from another random commit, to speed up compilation.
+          codeql-compile-${{ inputs.key }}-${{ github.ref_name }}- 
+          codeql-compile-${{ inputs.key }}-main-
+    - name: Fill compilation cache directory
+      id: fill-compilation-dir
+      shell: bash 
+      run: |
+        # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+        mkdir -p ${COMBINED_CACHE_DIR}
+        rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+        # copy the contents of the .cache folders into the combined cache folder.
+        cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+        # clean up the .cache folders
+        rm -rf **/.cache/*
+
+        echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
+      env: 
+        COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

.github/workflows/compile-queries.yml

@@ -14,58 +14,26 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      # calculate the merge-base with main, in a way that works both on PRs and pushes to main. 
-      - name: Calculate merge-base
-        if: ${{ github.event_name == 'pull_request' }}
-        env:
-          BASE_BRANCH: ${{ github.base_ref }}
-        run: |
-          MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
-          echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV
-      - name: Read CodeQL query compilation - PR
-        if: ${{ github.event_name == 'pull_request' }}
-        uses: actions/cache@v3
-        with:
-          path: '**/.cache'
-          key: codeql-compile-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
-          restore-keys: |
-            codeql-compile-${{ github.base_ref }}-${{ env.merge-base }}
-            codeql-compile-${{ github.base_ref }}-
-            codeql-compile-main-
-      - name: Fill CodeQL query compilation cache - main
-        if: ${{ github.event_name != 'pull_request' }}
-        uses: actions/cache@v3
-        with:
-          path: '**/.cache'
-          key: codeql-compile-${{ github.ref_name }}-${{ github.sha }} # just fill on main
-          restore-keys: | # restore from another random commit, to speed up compilation.
-            codeql-compile-${{ github.ref_name }}- 
-            codeql-compile-main-
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
         with:
           channel: 'release'
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: all-queries
       - name: check formatting
         run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
       - name: compile queries - check-only
         # run with --check-only if running in a PR (github.sha != main)
         if : ${{ github.event_name == 'pull_request' }}
         shell: bash
-        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only
+        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
       - name: compile queries - full
         # do full compile if running on main - this populates the cache
         if : ${{ github.event_name != 'pull_request' }}
         shell: bash
-        run: |
+        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-          # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-          mkdir -p ${COMBINED_CACHE_DIR}
-          rm -f */ql/{src,examples}/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-          # copy the contents of the .cache folders into the combined cache folder.
-          cp -r */ql/{src,examples}/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-          # clean up the .cache folders
-          rm -rf */ql/{src,examples}/.cache/*
-
-          # compile the queries
-          codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache ${COMBINED_CACHE_DIR}
         env: 
           COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

.github/workflows/ruby-qltest.yml

@@ -2,11 +2,7 @@ name: "Ruby: Run QL Tests"
 
 on:
   push:
-    paths:
+    # no path requirement on branch pushes, so the cache is more effective.
-      - "ruby/**"
-      - .github/workflows/ruby-qltest.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
@@ -48,17 +44,20 @@ jobs:
            xargs codeql execute upgrades testdb
           diff -q testdb/ruby.dbscheme downgrades/initial/ruby.dbscheme
   qltest:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-xl
     strategy:
       fail-fast: false
-      matrix:
-        slice: ["1/2", "2/2"]
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - uses: ./ruby/actions/create-extractor-pack
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: ruby-qltest
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 5000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test
+          codeql test run --threads=0 --ram 52000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}