Fix cli-integration tests

The main problem this commit fixes is with vscode 1.67.0, an error is
thrown when inside of integration tests and a dialog box is opened. We
were opening the telemetry dialog box. Now, an env variable is set
during cli-integration tests that prevents the dialog from being
opened.

There are also other cleanups and improvements with cli-integration
tests that assist with running locally:

- `vscode-test` dependency has been renamed to `@vscode/test-electron`,
  so use that instead and make the small API changes to support it.
- Commit the codeql-pack.lock.yml file so it isn't recreated on each
  test run.
- Ensure all databases are removed before _and after_ each test run
  that manipulates the set of installed databases
- Similarly, for quick query files, delete them before and after each
  test.
- Change some async `forEach` blocks to for loops in order to support
  sequential operations more easily.

.vscode/launch.json

@@ -12,7 +12,6 @@
         // Add a reference to a workspace to open. Eg-
         // "${workspaceRoot}/../vscode-codeql-starter/vscode-codeql-starter.code-workspace"
       ],
-      "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
@@ -46,7 +45,6 @@
         "ts-node/register",
         "test/pure-tests/**/*.ts"
       ],
-      "port": 9229,
       "stopOnEntry": false,
       "sourceMaps": true,
       "console": "integratedTerminal",
@@ -60,10 +58,10 @@
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/no-workspace/index",
+        "--disable-workspace-trust",
         "--disable-extensions",
         "--disable-gpu"
       ],
-      "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
@@ -77,11 +75,11 @@
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/minimal-workspace/index",
+        "--disable-workspace-trust",
         "--disable-extensions",
         "--disable-gpu",
         "${workspaceRoot}/extensions/ql-vscode/test/data"
       ],
-      "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
@@ -95,6 +93,7 @@
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/cli-integration/index",
+        "--disable-workspace-trust",
         "--disable-gpu",
         "--disable-extension",
         "eamodio.gitlens",
@@ -121,7 +120,6 @@
         // This option overrides the CLI_VERSION option.
         // "CLI_PATH": "${workspaceRoot}/../semmle-code/target/intree/codeql/codeql",
       },
-      "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",

extensions/ql-vscode/package.json

@@ -1112,6 +1112,7 @@
     "@octokit/rest": "^18.5.6",
     "@primer/octicons-react": "^16.3.0",
     "@primer/react": "^35.0.0",
+    "@vscode/test-electron": "^2.1.3",
     "child-process-promise": "^2.2.1",
     "classnames": "~2.2.6",
     "d3": "^6.3.1",
@@ -1209,7 +1210,6 @@
     "typescript": "^4.5.5",
     "typescript-formatter": "^7.2.2",
     "vsce": "^2.7.0",
-    "vscode-test": "^1.4.0",
     "webpack": "^5.28.0",
     "webpack-cli": "^4.6.0"
   },
@@ -1229,6 +1229,6 @@
     ]
   },
   "resolutions": {
-    "glob-parent": "~6.0.0"
+    "glob-parent": "6.0.0"
   }
 }

extensions/ql-vscode/src/config.ts

@@ -368,3 +368,7 @@ const ACTION_BRANCH = new Setting('actionBranch', REMOTE_QUERIES_SETTING);
 export function getActionBranch(): string {
   return ACTION_BRANCH.getValue<string>() || 'main';
 }
+
+export function isIntegrationTestMode() {
+  return process.env.INTEGRATION_TEST_MODE === 'true';
+}

extensions/ql-vscode/src/telemetry.ts

@@ -1,6 +1,6 @@
 import { ConfigurationTarget, Extension, ExtensionContext, ConfigurationChangeEvent } from 'vscode';
 import TelemetryReporter from 'vscode-extension-telemetry';
-import { ConfigListener, CANARY_FEATURES, ENABLE_TELEMETRY, GLOBAL_ENABLE_TELEMETRY, LOG_TELEMETRY } from './config';
+import { ConfigListener, CANARY_FEATURES, ENABLE_TELEMETRY, GLOBAL_ENABLE_TELEMETRY, LOG_TELEMETRY, isIntegrationTestMode } from './config';
 import * as appInsights from 'applicationinsights';
 import { logger } from './logging';
 import { UserCancellationException } from './commandRunner';
@@ -162,7 +162,11 @@ export class TelemetryListener extends ConfigListener {
     if (!this.wasTelemetryRequested()) {
       // if global telemetry is disabled, avoid showing the dialog or making any changes
       let result = undefined;
-      if (GLOBAL_ENABLE_TELEMETRY.getValue()) {
+      if (
+        GLOBAL_ENABLE_TELEMETRY.getValue() &&
+        // Avoid showing the dialog if we are in integration test mode.
+        !isIntegrationTestMode()
+      ) {
         // Extension won't start until this completes.
         result = await showBinaryChoiceWithUrlDialog(
           'Does the CodeQL Extension by GitHub have your permission to collect usage data and metrics to help us improve CodeQL for VSCode?',

extensions/ql-vscode/src/vscode-tests/cli-integration/data/codeql-pack.lock.yml

@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0

extensions/ql-vscode/src/vscode-tests/cli-integration/databases.test.ts

@@ -9,7 +9,7 @@ import { CodeQLCliServer } from '../../cli';
 import { DatabaseManager } from '../../databases';
 import { promptImportLgtmDatabase, importArchiveDatabase, promptImportInternetDatabase } from '../../databaseFetcher';
 import { ProgressCallback } from '../../commandRunner';
-import { dbLoc, DB_URL, storagePath } from './global.helper';
+import { cleanDatabases, dbLoc, DB_URL, storagePath } from './global.helper';
 
 /**
  * Run various integration tests for databases
@@ -27,6 +27,14 @@ describe('Databases', function() {
 
   beforeEach(async () => {
     try {
+      sandbox = sinon.createSandbox();
+      // the uri.fsPath function on windows returns a lowercase drive letter
+      // so, force the storage path string to be lowercase, too.
+      progressCallback = sandbox.spy();
+      inputBoxStub = sandbox.stub(window, 'showInputBox');
+      sandbox.stub(window, 'showErrorMessage');
+      sandbox.stub(window, 'showInformationMessage');
+
       const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
       if ('databaseManager' in extension) {
         databaseManager = extension.databaseManager;
@@ -34,19 +42,16 @@ describe('Databases', function() {
         throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
       }
 
-      sandbox = sinon.createSandbox();
+      await cleanDatabases(databaseManager);
-      // the uri.fsPath function on windows returns a lowercase drive letter
-      // so, force the storage path string to be lowercase, too.
-      progressCallback = sandbox.spy();
-      inputBoxStub = sandbox.stub(window, 'showInputBox');
     } catch (e) {
       fail(e as Error);
     }
   });
 
-  afterEach(() => {
+  afterEach(async () => {
     try {
       sandbox.restore();
+      await cleanDatabases(databaseManager);
     } catch (e) {
       fail(e as Error);
     }

extensions/ql-vscode/src/vscode-tests/cli-integration/global.helper.ts

@@ -4,13 +4,19 @@ import * as fs from 'fs-extra';
 import fetch from 'node-fetch';
 
 import { fail } from 'assert';
-import { ConfigurationTarget, extensions, workspace } from 'vscode';
+import { commands, ConfigurationTarget, extensions, workspace } from 'vscode';
 import { CodeQLExtensionInterface } from '../../extension';
+import { DatabaseManager } from '../../databases';
 
 // This file contains helpers shared between actual tests.
 
 export const DB_URL = 'https://github.com/github/vscode-codeql/files/5586722/simple-db.zip';
 
+process.addListener('unhandledRejection', (reason, p) => {
+  console.log('Unhandled Rejection at: Promise ', p, ' reason: ', reason);
+  fail(String(reason));
+});
+
 // We need to resolve the path, but the final three segments won't exist until later, so we only resolve the
 // first portion of the path.
 export const dbLoc = path.join(fs.realpathSync(path.join(__dirname, '../../../')), 'build/tests/db.zip');
@@ -84,3 +90,9 @@ export default function(mocha: Mocha) {
     }
   );
 }
+
+export async function cleanDatabases(databaseManager: DatabaseManager) {
+  for (const item of databaseManager.databaseItems) {
+    await commands.executeCommand('codeQLDatabases.removeDatabase', item);
+  }
+}

extensions/ql-vscode/src/vscode-tests/cli-integration/index.ts

@@ -1,4 +1,5 @@
 import 'source-map-support/register';
+import '@vscode/test-electron';
 import { runTestsInDirectory } from '../index-template';
 import 'mocha';
 import * as sinonChai from 'sinon-chai';
@@ -8,7 +9,6 @@ import * as chaiAsPromised from 'chai-as-promised';
 chai.use(chaiAsPromised);
 chai.use(sinonChai);
 
-// The simple database used throughout the tests
 export function run(): Promise<void> {
   return runTestsInDirectory(__dirname, true);
 }

extensions/ql-vscode/src/vscode-tests/cli-integration/packaging.test.ts

@@ -27,6 +27,17 @@ describe('Packaging commands', function() {
 
   beforeEach(async function() {
     sandbox = sinon.createSandbox();
+    progress = sandbox.spy();
+    quickPickSpy = sandbox.stub(window, 'showQuickPick');
+    inputBoxSpy = sandbox.stub(window, 'showInputBox');
+    showAndLogErrorMessageSpy = sandbox.stub();
+    showAndLogInformationMessageSpy = sandbox.stub();
+    mod = proxyquire('../../packaging', {
+      './helpers': {
+        showAndLogErrorMessage: showAndLogErrorMessageSpy,
+        showAndLogInformationMessage: showAndLogInformationMessageSpy,
+      },
+    });
 
     const extension = await extensions
       .getExtension<CodeQLExtensionInterface | Record<string, never>>(
@@ -45,17 +56,6 @@ describe('Packaging commands', function() {
         }. Skipping this test.`);
       this.skip();
     }
-    progress = sandbox.spy();
-    quickPickSpy = sandbox.stub(window, 'showQuickPick');
-    inputBoxSpy = sandbox.stub(window, 'showInputBox');
-    showAndLogErrorMessageSpy = sandbox.stub();
-    showAndLogInformationMessageSpy = sandbox.stub();
-    mod = proxyquire('../../packaging', {
-      './helpers': {
-        showAndLogErrorMessage: showAndLogErrorMessageSpy,
-        showAndLogInformationMessage: showAndLogInformationMessageSpy,
-      },
-    });
   });
 
   afterEach(() => {

extensions/ql-vscode/src/vscode-tests/cli-integration/queries.test.ts

@@ -8,7 +8,7 @@ import * as yaml from 'js-yaml';
 
 import { DatabaseItem, DatabaseManager } from '../../databases';
 import { CodeQLExtensionInterface } from '../../extension';
-import { dbLoc, storagePath } from './global.helper';
+import { cleanDatabases, dbLoc, storagePath } from './global.helper';
 import { importArchiveDatabase } from '../../databaseFetcher';
 import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo } from '../../run-queries';
 import { CodeQLCliServer } from '../../cli';
@@ -52,16 +52,21 @@ describe('Queries', function() {
         qs = extension.qs;
         cli.quiet = true;
         ctx = extension.ctx;
-        qlpackFile = `${ctx.storagePath}/quick-queries/qlpack.yml`;
+        qlpackFile = `${ctx.storageUri?.fsPath}/quick-queries/qlpack.yml`;
-        qlFile = `${ctx.storagePath}/quick-queries/quick-query.ql`;
+        qlFile = `${ctx.storageUri?.fsPath}/quick-queries/quick-query.ql`;
       } else {
         throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
       }
 
+      // Ensure we are starting from a clean slate.
+      safeDel(qlFile);
+      safeDel(qlpackFile);
+
       progress = sandbox.spy();
       token = {} as CancellationToken;
 
-      // Add a database
+      // Add a database, but make sure the database manager is empty first
+      await cleanDatabases(databaseManager);
       const uri = Uri.file(dbLoc);
       const maybeDbItem = await importArchiveDatabase(
         uri.toString(true),
@@ -81,9 +86,12 @@ describe('Queries', function() {
     }
   });
 
-  afterEach(() => {
+  afterEach(async () => {
     try {
       sandbox.restore();
+      safeDel(qlpackFile);
+      safeDel(qlFile);
+      await cleanDatabases(databaseManager);
     } catch (e) {
       fail(e as Error);
     }
@@ -135,9 +143,6 @@ describe('Queries', function() {
   });
 
   it('should create a quick query', async () => {
-    safeDel(qlFile);
-    safeDel(qlpackFile);
-
     await commands.executeCommand('codeQL.quickQuery');
 
     // should have created the quick query file and query pack file

extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts

@@ -9,6 +9,7 @@ import { skipIfNoCodeQL } from '../ensureCli';
 import { getOnDiskWorkspaceFolders, getQlPackForDbscheme, languageToDbScheme } from '../../helpers';
 import { resolveQueries } from '../../contextual/queryResolver';
 import { KeyType } from '../../contextual/keyType';
+import { fail } from 'assert';
 
 /**
  * Perform proper integration tests by running the CLI
@@ -74,24 +75,28 @@ describe('Use cli', function() {
 
   it('should resolve printAST queries for supported languages', async function() {
     skipIfNoCodeQL(this);
-    supportedLanguages.forEach(async lang => {
+    try {
-      if (lang === 'go') {
+      for (const lang of supportedLanguages) {
-        // The codeql-go submodule is not available in the integration tests.
+        if (lang === 'go') {
-        return;
+          // The codeql-go submodule is not available in the integration tests.
+          return;
+        }
+
+        console.log(`resolving printAST queries for ${lang}`);
+        const pack = await getQlPackForDbscheme(cli, languageToDbScheme[lang]);
+        expect(pack.dbschemePack).to.contain(lang);
+        if (pack.dbschemePackIsLibraryPack) {
+          expect(pack.queryPack).to.contain(lang);
+        }
+
+        const result = await resolveQueries(cli, pack, KeyType.PrintAstQuery);
+
+        // It doesn't matter what the name or path of the query is, only
+        // that we have found exactly one query.
+        expect(result.length).to.eq(1);
       }
-
+    } catch (e) {
-      console.log(`resolving printAST queries for ${lang}`);
+      fail(e as Error);
-      const pack = await getQlPackForDbscheme(cli, languageToDbScheme[lang]);
+    }
-      expect(pack.dbschemePack).to.contain(lang);
-      if (pack.dbschemePackIsLibraryPack) {
-        expect(pack.queryPack).to.contain(lang);
-      }
-
-      const result = await resolveQueries(cli, pack, KeyType.PrintAstQuery);
-
-      // It doesn't matter what the name or path of the query is, only
-      // that we have found exactly one query.
-      expect(result.length).to.eq(1);
-    });
   });
 });

extensions/ql-vscode/src/vscode-tests/minimal-workspace/config.test.ts

@@ -1,4 +1,3 @@
-import 'vscode-test';
 import * as Sinon from 'sinon';
 import { expect } from 'chai';
 import { workspace } from 'vscode';

extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts

@@ -1,4 +1,3 @@
-import 'vscode-test';
 import * as sinon from 'sinon';
 import * as tmp from 'tmp';
 import * as fs from 'fs-extra';

extensions/ql-vscode/src/vscode-tests/minimal-workspace/index.ts

@@ -1,10 +1,12 @@
 import 'source-map-support/register';
-import { runTestsInDirectory } from '../index-template';
+import '@vscode/test-electron';
-
 import * as sinonChai from 'sinon-chai';
 import * as chai from 'chai';
 import 'chai/register-should';
 import * as chaiAsPromised from 'chai-as-promised';
+
+import { runTestsInDirectory } from '../index-template';
+
 chai.use(chaiAsPromised);
 chai.use(sinonChai);
 

extensions/ql-vscode/src/vscode-tests/minimal-workspace/qltest-discovery.test.ts

@@ -1,4 +1,3 @@
-import 'vscode-test';
 import { Uri, WorkspaceFolder } from 'vscode';
 import { expect } from 'chai';
 import * as fs from 'fs-extra';

extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/fileRangeFromURI.test.ts

@@ -1,4 +1,3 @@
-import 'vscode-test';
 import { expect } from 'chai';
 import { Uri, Range } from 'vscode';
 

extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts

@@ -1,4 +1,3 @@
-import 'vscode-test';
 import * as yaml from 'js-yaml';
 import * as sinon from 'sinon';
 import { expect } from 'chai';

extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts

@@ -1,4 +1,3 @@
-import 'vscode-test';
 import * as sinon from 'sinon';
 import * as path from 'path';
 import * as fs from 'fs-extra';

extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts

@@ -1,4 +1,3 @@
-import 'vscode-test';
 import * as tmp from 'tmp';
 import * as path from 'path';
 import * as fs from 'fs-extra';

extensions/ql-vscode/src/vscode-tests/no-workspace/index.ts

@@ -1,10 +1,12 @@
 import 'source-map-support/register';
-import { runTestsInDirectory } from '../index-template';
+import '@vscode/test-electron';
 import * as sinonChai from 'sinon-chai';
 import * as chai from 'chai';
 import * as chaiAsPromised from 'chai-as-promised';
 import 'chai/register-should';
 
+import { runTestsInDirectory } from '../index-template';
+
 chai.use(chaiAsPromised);
 chai.use(sinonChai);
 

extensions/ql-vscode/src/vscode-tests/no-workspace/remote-queries/repository-selection.test.ts

@@ -1,4 +1,3 @@
-import 'vscode-test';
 import * as sinon from 'sinon';
 import { expect } from 'chai';
 import { window } from 'vscode';

extensions/ql-vscode/src/vscode-tests/no-workspace/test-adapter.test.ts

@@ -1,4 +1,3 @@
-import 'vscode-test';
 import * as sinon from 'sinon';
 import * as fs from 'fs-extra';
 import { Uri, WorkspaceFolder } from 'vscode';

extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts

@@ -5,18 +5,16 @@ import {
   runTests,
   downloadAndUnzipVSCode,
   resolveCliPathFromVSCodeExecutablePath
-} from 'vscode-test';
+} from '@vscode/test-electron';
 import { assertNever } from '../pure/helpers-pure';
+import * as tmp from 'tmp-promise';
 
-// For some reason, `TestOptions` is not exported directly from `vscode-test`,
+// For some reason, the following are not exported directly from `@vscode/test-electron`,
 // but we can be tricky and import directly from the out file.
-import { TestOptions } from 'vscode-test/out/runTest';
+import { TestOptions } from '@vscode/test-electron/out/runTest';
+import { systemDefaultPlatform } from '@vscode/test-electron/out/util';
 
 
-// Which version of vscode to test against. Can set to 'stable' or
-// 'insiders' or an explicit version number. See runTest.d.ts in
-// vscode-test for more details.
-
 // For CI purposes we want to leave this at 'stable' to catch any bugs
 // that might show up with new vscode versions released, even though
 // this makes testing not-quite-pure, but it can be changed for local
@@ -63,6 +61,8 @@ async function runTestsWithRetryOnSegfault(suite: TestOptions, tries: number): P
   process.exit(1);
 }
 
+const tmpDir = tmp.dirSync({ unsafeCleanup: true });
+
 /**
  * Integration test runner. Launches the VSCode Extension Development Host with this extension installed.
  * See https://github.com/microsoft/vscode-test/blob/master/sample/test/runTest.ts
@@ -76,10 +76,10 @@ async function main() {
     // Which tests to run. Use a comma-separated list of directories.
     const testDirsString = process.argv[2];
     const dirs = testDirsString.split(',').map(dir => dir.trim().toLocaleLowerCase());
-
+    const extensionTestsEnv: Record<string, string> = {};
     if (dirs.includes(TestDir.CliIntegration)) {
       console.log('Installing required extensions');
-      const cliPath = resolveCliPathFromVSCodeExecutablePath(vscodeExecutablePath);
+      const cliPath = resolveCliPathFromVSCodeExecutablePath(vscodeExecutablePath, systemDefaultPlatform);
       cp.spawnSync(
         cliPath,
         [
@@ -93,6 +93,7 @@ async function main() {
           stdio: 'inherit',
         }
       );
+      extensionTestsEnv.INTEGRATION_TEST_MODE = 'true';
     }
 
     console.log(`Running integration tests in these directories: ${dirs}`);
@@ -105,11 +106,13 @@ async function main() {
         vscodeExecutablePath,
         extensionDevelopmentPath,
         extensionTestsPath: path.resolve(__dirname, dir, 'index'),
+        extensionTestsEnv,
         launchArgs
       }, 3);
     }
   } catch (err) {
     console.error(`Unexpected exception while running tests: ${err}`);
+    console.error((err as Error).stack);
     process.exit(1);
   }
 }
@@ -122,13 +125,15 @@ function getLaunchArgs(dir: TestDir) {
     case TestDir.NoWorksspace:
       return [
         '--disable-extensions',
-        '--disable-gpu'
+        '--disable-gpu',
+        '--user-data-dir=' + path.join(tmpDir.name, dir, 'user-data')
       ];
 
     case TestDir.MinimalWorksspace:
       return [
         '--disable-extensions',
         '--disable-gpu',
+        '--user-data-dir=' + path.join(tmpDir.name, dir, 'user-data'),
         path.resolve(__dirname, '../../test/data')
       ];
 
@@ -145,8 +150,8 @@ function getLaunchArgs(dir: TestDir) {
         'github.codespaces',
         '--disable-extension',
         'github.copilot',
-        process.env.TEST_CODEQL_PATH!
+        '--user-data-dir=' + path.join(tmpDir.name, dir, 'user-data'),
-      ];
+      ].concat(process.env.TEST_CODEQL_PATH ? [process.env.TEST_CODEQL_PATH] : []);
 
     default:
       assertNever(dir);

extensions/ql-vscode/test/pure-tests/sarif-processing.test.ts

@@ -1,4 +1,3 @@
-import 'vscode-test';
 import 'mocha';
 import * as chaiAsPromised from 'chai-as-promised';
 import * as chai from 'chai';