Split out CLI version tests to separate workflow (#2729)

2023-08-22 11:28:36 +01:00
parent 13ce5f7e3c
commit 3094405f68
3 changed files with 121 additions and 39 deletions
--- a/.github/workflows/cli-test.yml
+++ b/.github/workflows/cli-test.yml
@@ -0,0 +1,109 @@
+name: Run CLI tests
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * *'
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    paths:
+      - extensions/ql-vscode/src/codeql-cli/**
+      - extensions/ql-vscode/src/language-support/**
+      - extensions/ql-vscode/src/query-server/**
+
+jobs:
+  find-nightly:
+    name: Find Nightly Release
+    runs-on: ubuntu-latest
+    outputs:
+      url: ${{ steps.get-url.outputs.nightly-url }}
+    steps:
+      - name: Get Nightly Release URL
+        id: get-url
+        env:
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+        shell: bash
+        # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
+        run: |
+          LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'`
+          echo "nightly-url=https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST" >> "$GITHUB_OUTPUT"
+
+  set-matrix:
+    name: Set Matrix for cli-test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set the variables
+        id: set-variables
+        run: echo "cli-versions=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc)" >> $GITHUB_OUTPUT
+    outputs:
+      cli-versions: ${{ steps.set-variables.outputs.cli-versions }}
+
+  cli-test:
+    name: CLI Test
+    runs-on: ${{ matrix.os }}
+    needs: [find-nightly, set-matrix]
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        version: ${{ fromJson(needs.set-matrix.outputs.cli-versions) }} 
+      fail-fast: false
+    env:
+      CLI_VERSION: ${{ matrix.version }}
+      NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
+      TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16.17.1'
+          cache: 'npm'
+          cache-dependency-path: extensions/ql-vscode/package-lock.json
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: |
+          npm ci
+        shell: bash
+
+      - name: Build
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run build
+        shell: bash
+
+      - name: Decide on ref of CodeQL repo
+        id: choose-ref
+        shell: bash
+        run: |
+          if [[ "${{ matrix.version }}" == "nightly" ]]
+          then
+            REF="codeql-cli/latest"
+          else
+            REF="codeql-cli/${{ matrix.version }}"
+          fi
+          echo "ref=$REF" >> "$GITHUB_OUTPUT"
+
+      - name: Checkout QL
+        uses: actions/checkout@v3
+        with:
+          repository: github/codeql
+          ref: ${{ steps.choose-ref.outputs.ref }}
+          path: codeql
+
+      - name: Run CLI tests (Linux)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          unset DBUS_SESSION_BUS_ADDRESS
+          /usr/bin/xvfb-run npm run test:cli-integration
+
+      - name: Run CLI tests (Windows)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'windows-latest'
+        run: |
+          npm run test:cli-integration
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -53,22 +53,6 @@ jobs:
          name: vscode-codeql-extension
          path: artifacts

-  find-nightly:
-    name: Find Nightly Release
-    runs-on: ubuntu-latest
-    outputs:
-      url: ${{ steps.get-url.outputs.nightly-url }}
-    steps:
-      - name: Get Nightly Release URL
-        id: get-url
-        env:
-          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
-        shell: bash
-        # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
-        run: |
-          LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'`
-          echo "nightly-url=https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST" >> "$GITHUB_OUTPUT"
-
  lint:
    name: Lint
    runs-on: ubuntu-latest
@@ -198,31 +182,31 @@ jobs:
        run: |
          npm run test:vscode-integration

-  set-matrix:
-    name: Set Matrix for cli-test
+  get-latest-cli-version:
+    name: Get latest CLI version
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
-      - name: Set the variables
-        id: set-variables
-        run: echo "cli-versions=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc)" >> $GITHUB_OUTPUT
+      - name: Set the variable
+        id: set-variable
+        run: |
+          echo "cli-version=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc '.[0]')" >> $GITHUB_OUTPUT
+          echo "$cli-version"
    outputs:
-      cli-versions: ${{ steps.set-variables.outputs.cli-versions }}
+      cli-version: ${{ steps.set-variable.outputs.cli-version }}

  cli-test:
    name: CLI Test
    runs-on: ${{ matrix.os }}
-    needs: [find-nightly, set-matrix]
+    needs: [get-latest-cli-version]
    timeout-minutes: 30
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest]
-        version: ${{ fromJson(needs.set-matrix.outputs.cli-versions) }}
      fail-fast: false
    env:
-      CLI_VERSION: ${{ matrix.version }}
-      NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
+      CLI_VERSION: ${{ needs.get-latest-cli-version.outputs.cli-version }}
      TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'

    steps:
@@ -247,23 +231,11 @@ jobs:
          npm run build
        shell: bash

-      - name: Decide on ref of CodeQL repo
-        id: choose-ref
-        shell: bash
-        run: |
-          if [[ "${{ matrix.version }}" == "nightly" ]]
-          then
-            REF="codeql-cli/latest"
-          else
-            REF="codeql-cli/${{ matrix.version }}"
-          fi
-          echo "ref=$REF" >> "$GITHUB_OUTPUT"
-
      - name: Checkout QL
        uses: actions/checkout@v3
        with:
          repository: github/codeql
-          ref: ${{ steps.choose-ref.outputs.ref }}
+          ref: 'codeql-cli/${{ needs.get-latest-cli-version.outputs.cli-version }}'
          path: codeql

      - name: Run CLI tests (Linux)
--- a/docs/releasing.md
+++ b/docs/releasing.md
@@ -1,5 +1,6 @@
 # Releasing (write access required)

+1. Run the ["Run CLI tests" workflow](https://github.com/github/vscode-codeql/actions/workflows/cli-test.yml) and make sure the tests are green. If there were no merges between the time the workflow ran (it runs daily), and the release, you can skip this step.  
 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
    * Go through all recent PRs and make sure they are properly accounted for.
    * Make sure all changelog entries have links back to their PR(s) if appropriate.