Merge pull request #2058 from github/koesie10/improve-test-setup

Improve test setup

.github/workflows/main.yml

@@ -132,7 +132,12 @@ jobs:
       - name: Run unit tests
         working-directory: extensions/ql-vscode
         run: |
-          npm run test
+          npm run test:unit
+
+      - name: Run view tests
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run test:view
 
   test:
     name: Test
@@ -173,7 +178,7 @@ jobs:
           VSCODE_CODEQL_GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
         run: |
           unset DBUS_SESSION_BUS_ADDRESS
-          /usr/bin/xvfb-run npm run integration
+          /usr/bin/xvfb-run npm run test:vscode-integration
 
       - name: Run integration tests (Windows)
         if: matrix.os == 'windows-latest'
@@ -181,7 +186,7 @@ jobs:
         env:
           VSCODE_CODEQL_GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
         run: |
-          npm run integration
+          npm run test:vscode-integration
 
   set-matrix:
     name: Set Matrix for cli-test
@@ -254,10 +259,10 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         run: |
           unset DBUS_SESSION_BUS_ADDRESS
-          /usr/bin/xvfb-run npm run cli-integration
+          /usr/bin/xvfb-run npm run test:cli-integration
 
       - name: Run CLI tests (Windows)
         working-directory: extensions/ql-vscode
         if: matrix.os == 'windows-latest'
         run: |
-          npm run cli-integration
+          npm run test:cli-integration

CONTRIBUTING.md

@@ -98,10 +98,11 @@ We have several types of tests:
 * Unit tests: these live in the `tests/unit-tests/` directory
 * View tests: these live in `src/view/variant-analysis/__tests__/`
 * VSCode integration tests:
-  * `test/vscode-tests/no-workspace` tests: These are intended to cover functionality that is meant to work before you even have a workspace open. 
+  * `test/vscode-tests/activated-extension` tests: These are intended to cover functionality that require the full extension to be activated but don't require the CLI. This suite is not run against multiple versions of the CLI in CI.
+  * `test/vscode-tests/no-workspace` tests: These are intended to cover functionality around not having a workspace. The extension is not activated in these tests.
   * `test/vscode-tests/minimal-workspace` tests: These are intended to cover functionality that need a workspace but don't require the full extension to be activated.
 * CLI integration tests: these live in `test/vscode-tests/cli-integration`
-  * These tests are intendended to be cover functionality that is related to the integration between the CodeQL CLI and the extension.
+  * These tests are intended to cover functionality that is related to the integration between the CodeQL CLI and the extension. These tests are run against each supported versions of the CLI in CI.
 
 The CLI integration tests require an instance of the CodeQL CLI to run so they will require some extra setup steps. When adding new tests to our test suite, please be mindful of whether they need to be in the cli-integration folder. If the tests don't depend on the CLI, they are better suited to being a VSCode integration test.
 
@@ -119,7 +120,7 @@ Then, from the `extensions/ql-vscode` directory, use the appropriate command to
 
 * Unit tests: `npm run test:unit`
 * View Tests: `npm test:view`
-* VSCode integration tests: `npm run integration`
+* VSCode integration tests: `npm run test:vscode-integration`
 
 ###### CLI integration tests
 
@@ -130,7 +131,7 @@ The CLI integration tests require the CodeQL standard libraries in order to run
 2. Run your test command:
 
 ```shell
-cd extensions/ql-vscode && npm run cli-integration
+cd extensions/ql-vscode && npm run test:cli-integration
 ```
 
 ##### 2. From VSCode
@@ -161,13 +162,13 @@ The easiest way to run a single test is to change the `it` of the test to `it.on
 to only run tests for this specific file. For example, to run the test `test/vscode-tests/cli-integration/run-queries.test.ts`:
 
 ```shell
-npm run cli-integration -- --runTestsByPath test/vscode-tests/cli-integration/run-queries.test.ts
+npm run test:cli-integration -- --runTestsByPath test/vscode-tests/cli-integration/run-queries.test.ts
 ```
 
 You can also use the `--testNamePattern` option to run a specific test within a file. For example, to run the test `test/vscode-tests/cli-integration/run-queries.test.ts`:
 
 ```shell
-npm run cli-integration -- --runTestsByPath test/vscode-tests/cli-integration/run-queries.test.ts --testNamePattern "should create a QueryEvaluationInfo"
+npm run test:cli-integration -- --runTestsByPath test/vscode-tests/cli-integration/run-queries.test.ts --testNamePattern "should create a QueryEvaluationInfo"
 ```
 
 ##### 2. From VSCode

extensions/ql-vscode/jest.config.js

@@ -8,6 +8,7 @@ module.exports = {
   projects: [
     "<rootDir>/src/view",
     "<rootDir>/test/unit-tests",
+    "<rootDir>/test/vscode-tests/activated-extension",
     "<rootDir>/test/vscode-tests/cli-integration",
     "<rootDir>/test/vscode-tests/no-workspace",
     "<rootDir>/test/vscode-tests/minimal-workspace",

extensions/ql-vscode/package.json

@@ -1326,13 +1326,14 @@
   "scripts": {
     "build": "gulp",
     "watch": "gulp watch",
-    "test": "npm-run-all -p test:*",
+    "test": "npm-run-all test:*",
     "test:unit": "cross-env TZ=UTC LANG=en-US jest --projects test/unit-tests",
     "test:view": "jest --projects src/view",
-    "integration": "npm-run-all integration:*",
-    "integration:no-workspace": "jest --projects test/vscode-tests/no-workspace",
-    "integration:minimal-workspace": "jest --projects test/vscode-tests/minimal-workspace",
-    "cli-integration": "jest --projects test/vscode-tests/cli-integration",
+    "test:vscode-integration": "npm-run-all test:vscode-integration:*",
+    "test:vscode-integration:activated-extension": "jest --projects test/vscode-tests/activated-extension",
+    "test:vscode-integration:no-workspace": "jest --projects test/vscode-tests/no-workspace",
+    "test:vscode-integration:minimal-workspace": "jest --projects test/vscode-tests/minimal-workspace",
+    "test:cli-integration": "jest --projects test/vscode-tests/cli-integration",
     "update-vscode": "node ./node_modules/vscode/bin/install",
     "format": "prettier --write **/*.{ts,tsx} && eslint . --ext .ts,.tsx --fix",
     "lint": "eslint . --ext .js,.ts,.tsx --max-warnings=0",

extensions/ql-vscode/test/vscode-tests/activated-extension/jest-runner-vscode.config.js

@@ -0,0 +1,29 @@
+const path = require("path");
+
+const {
+  config: baseConfig,
+  rootDir,
+} = require("../jest-runner-vscode.config.base");
+
+/** @type import("jest-runner-vscode").RunnerOptions */
+const config = {
+  ...baseConfig,
+  launchArgs: [
+    ...(baseConfig.launchArgs ?? []),
+    // explicitly disable extensions that are known to interfere with the CLI integration tests
+    "--disable-extension",
+    "eamodio.gitlens",
+    "--disable-extension",
+    "github.codespaces",
+    "--disable-extension",
+    "github.copilot",
+    path.resolve(rootDir, "test/data"),
+  ],
+  extensionTestsEnv: {
+    ...baseConfig.extensionTestsEnv,
+    INTEGRATION_TEST_MODE: "true",
+  },
+  retries: 3,
+};
+
+module.exports = config;

extensions/ql-vscode/test/vscode-tests/activated-extension/jest.config.ts

@@ -0,0 +1,11 @@
+import type { Config } from "jest";
+
+import baseConfig from "../jest.config.base";
+
+const config: Config = {
+  ...baseConfig,
+  runner: "<rootDir>/../jest-runner-installed-extensions.ts",
+  setupFilesAfterEnv: ["<rootDir>/jest.setup.ts"],
+};
+
+export default config;

extensions/ql-vscode/test/vscode-tests/activated-extension/jest.setup.ts

@@ -0,0 +1,12 @@
+import {
+  beforeAllAction,
+  beforeEachAction,
+} from "../jest.activated-extension.setup";
+
+beforeAll(async () => {
+  await beforeAllAction();
+});
+
+beforeEach(async () => {
+  await beforeEachAction();
+});

extensions/ql-vscode/test/vscode-tests/cli-integration/databaseFetcher.test.ts

@@ -8,7 +8,7 @@ import {
   importArchiveDatabase,
   promptImportInternetDatabase,
 } from "../../../src/databaseFetcher";
-import { cleanDatabases, dbLoc, DB_URL, storagePath } from "./global.helper";
+import { cleanDatabases, dbLoc, DB_URL, storagePath } from "../global.helper";
 
 jest.setTimeout(60_000);
 

extensions/ql-vscode/test/vscode-tests/cli-integration/jest.config.ts

@@ -4,7 +4,7 @@ import baseConfig from "../jest.config.base";
 
 const config: Config = {
   ...baseConfig,
-  runner: "<rootDir>/jest-runner-cli-integration.ts",
+  runner: "<rootDir>/../jest-runner-installed-extensions.ts",
   setupFilesAfterEnv: ["<rootDir>/jest.setup.ts"],
 };
 

extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts

@@ -1,28 +1,24 @@
+import { workspace } from "vscode";
+
+import {
+  beforeAllAction,
+  beforeEachAction,
+} from "../jest.activated-extension.setup";
+import * as tmp from "tmp";
 import {
-  mkdirpSync,
-  existsSync,
   createWriteStream,
+  existsSync,
+  mkdirpSync,
   realpathSync,
 } from "fs-extra";
 import { dirname } from "path";
+import { DB_URL, dbLoc, setStoragePath, storagePath } from "../global.helper";
 import fetch from "node-fetch";
-import { DB_URL, dbLoc, setStoragePath, storagePath } from "./global.helper";
-import * as tmp from "tmp";
-import { CUSTOM_CODEQL_PATH_SETTING } from "../../../src/config";
-import { ConfigurationTarget, env, extensions, workspace } from "vscode";
-import { beforeEachAction } from "../test-config";
 
 // create an extension storage location
 let removeStorage: tmp.DirResult["removeCallback"] | undefined;
 
 beforeAll(async () => {
-  // Set the CLI version here before activation to ensure we don't accidentally try to download a cli
-  await beforeEachAction();
-  await CUSTOM_CODEQL_PATH_SETTING.updateValue(
-    process.env.CLI_PATH,
-    ConfigurationTarget.Workspace,
-  );
-
   // ensure the test database is downloaded
   mkdirpSync(dirname(dbLoc));
   if (!existsSync(dbLoc)) {
@@ -54,6 +50,14 @@ beforeAll(async () => {
 
   removeStorage = dir.removeCallback;
 
+  await beforeAllAction();
+});
+
+beforeEach(async () => {
+  await beforeEachAction();
+});
+
+beforeAll(() => {
   // check that the codeql folder is found in the workspace
   const folders = workspace.workspaceFolders;
   if (!folders) {
@@ -70,20 +74,6 @@ beforeAll(async () => {
       );
     }
   }
-
-  // Activate the extension
-  await extensions.getExtension("GitHub.vscode-codeql")?.activate();
-});
-
-beforeEach(async () => {
-  jest.spyOn(env, "openExternal").mockResolvedValue(false);
-
-  await beforeEachAction();
-
-  await CUSTOM_CODEQL_PATH_SETTING.updateValue(
-    process.env.CLI_PATH,
-    ConfigurationTarget.Workspace,
-  );
 });
 
 // ensure extension is cleaned up.

extensions/ql-vscode/test/vscode-tests/cli-integration/new-query.test.ts

@@ -11,7 +11,7 @@ import { describeWithCodeQL } from "../cli";
 import { QueryServerClient } from "../../../src/query-server/queryserver-client";
 import { extLogger, ProgressReporter } from "../../../src/common";
 import { QueryResultType } from "../../../src/pure/new-messages";
-import { cleanDatabases, dbLoc, storagePath } from "./global.helper";
+import { cleanDatabases, dbLoc, storagePath } from "../global.helper";
 import { importArchiveDatabase } from "../../../src/databaseFetcher";
 
 const baseDir = join(__dirname, "../../../test/data");

extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts

@@ -17,7 +17,7 @@ import { load, dump } from "js-yaml";
 
 import { DatabaseItem, DatabaseManager } from "../../../src/databases";
 import { CodeQLExtensionInterface } from "../../../src/extension";
-import { cleanDatabases, dbLoc, storagePath } from "./global.helper";
+import { cleanDatabases, dbLoc, storagePath } from "../global.helper";
 import { importArchiveDatabase } from "../../../src/databaseFetcher";
 import { CodeQLCliServer } from "../../../src/cli";
 import { describeWithCodeQL } from "../cli";

extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/variant-analysis-manager.test.ts

@@ -31,7 +31,7 @@ import {
   fixWorkspaceReferences,
   restoreWorkspaceReferences,
   storagePath,
-} from "../global.helper";
+} from "../../global.helper";
 import { VariantAnalysisResultsManager } from "../../../../src/remote-queries/variant-analysis-results-manager";
 import { createMockVariantAnalysis } from "../../../factories/remote-queries/shared/variant-analysis";
 import * as VariantAnalysisModule from "../../../../src/remote-queries/shared/variant-analysis";

extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/variant-analysis-results-manager.test.ts

@@ -9,7 +9,7 @@ import * as fetchModule from "node-fetch";
 
 import { VariantAnalysisResultsManager } from "../../../../src/remote-queries/variant-analysis-results-manager";
 import { CodeQLCliServer } from "../../../../src/cli";
-import { storagePath } from "../global.helper";
+import { storagePath } from "../../global.helper";
 import { faker } from "@faker-js/faker";
 import { createMockVariantAnalysisRepositoryTask } from "../../../factories/remote-queries/shared/variant-analysis-repo-tasks";
 import {

extensions/ql-vscode/test/vscode-tests/global.helper.ts

@@ -2,11 +2,11 @@ import { join } from "path";
 import { load, dump } from "js-yaml";
 import { realpathSync, readFileSync, writeFileSync } from "fs-extra";
 import { commands } from "vscode";
-import { DatabaseManager } from "../../../src/databases";
-import { CodeQLCliServer } from "../../../src/cli";
-import { removeWorkspaceRefs } from "../../../src/remote-queries/run-remote-query";
+import { DatabaseManager } from "../../src/databases";
+import { CodeQLCliServer } from "../../src/cli";
+import { removeWorkspaceRefs } from "../../src/remote-queries/run-remote-query";
 
-// This file contains helpers shared between actual tests.
+// This file contains helpers shared between tests that work with an activated extension.
 
 export const DB_URL =
   "https://github.com/github/vscode-codeql/files/5586722/simple-db.zip";

extensions/ql-vscode/test/vscode-tests/jest-runner-installed-extensions.ts

@@ -8,9 +8,9 @@ import {
   downloadAndUnzipVSCode,
   resolveCliArgsFromVSCodeExecutablePath,
 } from "@vscode/test-electron";
-import { ensureCli } from "../ensureCli";
+import { ensureCli } from "./ensureCli";
 
-export default class JestRunnerCliIntegration extends VSCodeTestRunner {
+export default class JestRunnerInstalledExtensions extends VSCodeTestRunner {
   async runTests(
     tests: JestRunner.Test[],
     watcher: JestRunner.TestWatcher,

extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts

@@ -0,0 +1,26 @@
+import { CUSTOM_CODEQL_PATH_SETTING } from "../../src/config";
+import { ConfigurationTarget, env, extensions } from "vscode";
+import { beforeEachAction as testConfigBeforeEachAction } from "./test-config";
+
+export async function beforeAllAction() {
+  // Set the CLI version here before activation to ensure we don't accidentally try to download a cli
+  await testConfigBeforeEachAction();
+  await CUSTOM_CODEQL_PATH_SETTING.updateValue(
+    process.env.CLI_PATH,
+    ConfigurationTarget.Workspace,
+  );
+
+  // Activate the extension
+  await extensions.getExtension("GitHub.vscode-codeql")?.activate();
+}
+
+export async function beforeEachAction() {
+  jest.spyOn(env, "openExternal").mockResolvedValue(false);
+
+  await testConfigBeforeEachAction();
+
+  await CUSTOM_CODEQL_PATH_SETTING.updateValue(
+    process.env.CLI_PATH,
+    ConfigurationTarget.Workspace,
+  );
+}