Split out CLI version tests to separate workflow (#2729)

.github/workflows/cli-test.yml

@@ -0,0 +1,109 @@
+name: Run CLI tests
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * *'
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    paths:
+      - extensions/ql-vscode/src/codeql-cli/**
+      - extensions/ql-vscode/src/language-support/**
+      - extensions/ql-vscode/src/query-server/**
+
+jobs:
+  find-nightly:
+    name: Find Nightly Release
+    runs-on: ubuntu-latest
+    outputs:
+      url: ${{ steps.get-url.outputs.nightly-url }}
+    steps:
+      - name: Get Nightly Release URL
+        id: get-url
+        env:
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+        shell: bash
+        # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
+        run: |
+          LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'`
+          echo "nightly-url=https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST" >> "$GITHUB_OUTPUT"
+
+  set-matrix:
+    name: Set Matrix for cli-test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set the variables
+        id: set-variables
+        run: echo "cli-versions=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc)" >> $GITHUB_OUTPUT
+    outputs:
+      cli-versions: ${{ steps.set-variables.outputs.cli-versions }}
+
+  cli-test:
+    name: CLI Test
+    runs-on: ${{ matrix.os }}
+    needs: [find-nightly, set-matrix]
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        version: ${{ fromJson(needs.set-matrix.outputs.cli-versions) }} 
+      fail-fast: false
+    env:
+      CLI_VERSION: ${{ matrix.version }}
+      NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
+      TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16.17.1'
+          cache: 'npm'
+          cache-dependency-path: extensions/ql-vscode/package-lock.json
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: |
+          npm ci
+        shell: bash
+
+      - name: Build
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run build
+        shell: bash
+
+      - name: Decide on ref of CodeQL repo
+        id: choose-ref
+        shell: bash
+        run: |
+          if [[ "${{ matrix.version }}" == "nightly" ]]
+          then
+            REF="codeql-cli/latest"
+          else
+            REF="codeql-cli/${{ matrix.version }}"
+          fi
+          echo "ref=$REF" >> "$GITHUB_OUTPUT"
+
+      - name: Checkout QL
+        uses: actions/checkout@v3
+        with:
+          repository: github/codeql
+          ref: ${{ steps.choose-ref.outputs.ref }}
+          path: codeql
+
+      - name: Run CLI tests (Linux)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          unset DBUS_SESSION_BUS_ADDRESS
+          /usr/bin/xvfb-run npm run test:cli-integration
+
+      - name: Run CLI tests (Windows)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'windows-latest'
+        run: |
+          npm run test:cli-integration

.github/workflows/main.yml

@@ -53,22 +53,6 @@ jobs:
           name: vscode-codeql-extension
           path: artifacts
 
-  find-nightly:
-    name: Find Nightly Release
-    runs-on: ubuntu-latest
-    outputs:
-      url: ${{ steps.get-url.outputs.nightly-url }}
-    steps:
-      - name: Get Nightly Release URL
-        id: get-url
-        env:
-          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
-        shell: bash
-        # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
-        run: |
-          LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'`
-          echo "nightly-url=https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST" >> "$GITHUB_OUTPUT"
-
   lint:
     name: Lint
     runs-on: ubuntu-latest
@@ -198,31 +182,31 @@ jobs:
         run: |
           npm run test:vscode-integration
 
-  set-matrix:
+  get-latest-cli-version:
-    name: Set Matrix for cli-test
+    name: Get latest CLI version
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Set the variables
+      - name: Set the variable
-        id: set-variables
+        id: set-variable
-        run: echo "cli-versions=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc)" >> $GITHUB_OUTPUT
+        run: |
+          echo "cli-version=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc '.[0]')" >> $GITHUB_OUTPUT
+          echo "$cli-version"
     outputs:
-      cli-versions: ${{ steps.set-variables.outputs.cli-versions }}
+      cli-version: ${{ steps.set-variable.outputs.cli-version }}
 
   cli-test:
     name: CLI Test
     runs-on: ${{ matrix.os }}
-    needs: [find-nightly, set-matrix]
+    needs: [get-latest-cli-version]
     timeout-minutes: 30
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        version: ${{ fromJson(needs.set-matrix.outputs.cli-versions) }}
       fail-fast: false
     env:
-      CLI_VERSION: ${{ matrix.version }}
+      CLI_VERSION: ${{ needs.get-latest-cli-version.outputs.cli-version }}
-      NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
       TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'
 
     steps:
@@ -247,23 +231,11 @@ jobs:
           npm run build
         shell: bash
 
-      - name: Decide on ref of CodeQL repo
-        id: choose-ref
-        shell: bash
-        run: |
-          if [[ "${{ matrix.version }}" == "nightly" ]]
-          then
-            REF="codeql-cli/latest"
-          else
-            REF="codeql-cli/${{ matrix.version }}"
-          fi
-          echo "ref=$REF" >> "$GITHUB_OUTPUT"
-
       - name: Checkout QL
         uses: actions/checkout@v3
         with:
           repository: github/codeql
-          ref: ${{ steps.choose-ref.outputs.ref }}
+          ref: 'codeql-cli/${{ needs.get-latest-cli-version.outputs.cli-version }}'
           path: codeql
 
       - name: Run CLI tests (Linux)

docs/releasing.md

@@ -1,5 +1,6 @@
 # Releasing (write access required)
 
+1. Run the ["Run CLI tests" workflow](https://github.com/github/vscode-codeql/actions/workflows/cli-test.yml) and make sure the tests are green. If there were no merges between the time the workflow ran (it runs daily), and the release, you can skip this step.  
 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
     * Go through all recent PRs and make sure they are properly accounted for.
     * Make sure all changelog entries have links back to their PR(s) if appropriate.