Move query.test.ts to the cli integration tests

* Now query.test.ts runs on multiple cli versions
* Removed most `dispose` calls in cli tests because each test shares the
  same instance of the extension and all of its properties. So, we
  shouldn't be disposing until the last test completes. It's likely that
  we will need to be more careful about cleaning up state between test
  runs, but we haven't hit that yet and this can happen in a later
  commit.

.github/workflows/main.yml

@@ -124,7 +124,7 @@ jobs:
         version: ['v2.2.6', 'v2.3.3', 'v2.4.0']
     env:
       CLI_VERSION: ${{ matrix.version }}
-      QL_PATH: '${{ github.workspace }}/codeql'
+      TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'
 
     steps:
       - name: Checkout

extensions/ql-vscode/src/extension.ts

@@ -124,6 +124,7 @@ export interface CodeQLExtensionInterface {
   readonly distributionManager: DistributionManager;
   readonly databaseManager: DatabaseManager;
   readonly databaseUI: DatabaseUI;
+  readonly dispose: () => void;
 }
 
 /**
@@ -679,7 +680,10 @@ async function activateWithInstalledDistribution(
     qs,
     distributionManager,
     databaseManager: dbm,
-    databaseUI
+    databaseUI,
+    dispose: () => {
+      ctx.subscriptions.forEach(d => d.dispose());
+    }
   };
 }
 

extensions/ql-vscode/src/vscode-tests/cli-integration/databases.test.ts

@@ -2,8 +2,7 @@ import * as sinon from 'sinon';
 import * as path from 'path';
 import { fail } from 'assert';
 import { expect } from 'chai';
-import { extensions, CancellationToken, Uri } from 'vscode';
-import * as vscode from 'vscode';
+import { extensions, CancellationToken, Uri, window } from 'vscode';
 
 import { CodeQLExtensionInterface } from '../../extension';
 import { DatabaseManager } from '../../databases';
@@ -37,7 +36,7 @@ describe('Databases', function() {
       // the uri.fsPath function on windows returns a lowercase drive letter
       // so, force the storage path string to be lowercase, too.
       progressCallback = sandbox.spy();
-      inputBoxStub = sandbox.stub(vscode.window, 'showInputBox');
+      inputBoxStub = sandbox.stub(window, 'showInputBox');
     } catch (e) {
       fail(e);
     }
@@ -45,7 +44,7 @@ describe('Databases', function() {
 
   afterEach(() => {
     try {
-      databaseManager.dispose();
+      // dispose();
       sandbox.restore();
     } catch (e) {
       fail(e);

extensions/ql-vscode/src/vscode-tests/cli-integration/global.helper.ts

@@ -4,7 +4,8 @@ import * as fs from 'fs-extra';
 import fetch from 'node-fetch';
 
 import { fail } from 'assert';
-import { ConfigurationTarget, workspace } from 'vscode';
+import { ConfigurationTarget, extensions, workspace } from 'vscode';
+import { CodeQLExtensionInterface } from '../../extension';
 
 // This file contains helpers shared between actual tests.
 
@@ -21,48 +22,57 @@ export default function(mocha: /*Mocha*/ any) {
   // create an extension storage location
   let removeStorage: tmp.DirResult['removeCallback'] | undefined;
 
-  // ensure the test database is downloaded
-  mocha.globalSetup([async () => {
-    fs.mkdirpSync(path.dirname(dbLoc));
-    if (!fs.existsSync(dbLoc)) {
-      try {
-        await new Promise((resolve, reject) => {
-          fetch(DB_URL).then(response => {
-            const dest = fs.createWriteStream(dbLoc);
-            response.body.pipe(dest);
+  mocha.globalSetup([
+    // ensure the test database is downloaded
+    async () => {
+      fs.mkdirpSync(path.dirname(dbLoc));
+      if (!fs.existsSync(dbLoc)) {
+        try {
+          await new Promise((resolve, reject) => {
+            fetch(DB_URL).then(response => {
+              const dest = fs.createWriteStream(dbLoc);
+              response.body.pipe(dest);
 
-            response.body.on('error', reject);
-            dest.on('error', reject);
-            dest.on('close', () => {
-              resolve(dbLoc);
+              response.body.on('error', reject);
+              dest.on('error', reject);
+              dest.on('close', () => {
+                resolve(dbLoc);
+              });
             });
           });
-        });
-      } catch (e) {
-        fail('Failed to download test database: ' + e);
+        } catch (e) {
+          fail('Failed to download test database: ' + e);
+        }
       }
+    },
+
+    // Set the CLI version here before activation to ensure we don't accidentally try to download a cli
+    async () => {
+      await workspace.getConfiguration().update('codeQL.cli.executablePath', process.env.CLI_PATH, ConfigurationTarget.Global);
+    },
+
+    // Create the temp directory to be used as extension local storage.
+    () => {
+      const dir = tmp.dirSync();
+      storagePath = fs.realpathSync(dir.name);
+      if (storagePath.substring(0, 2).match(/[A-Z]:/)) {
+        storagePath = storagePath.substring(0, 1).toLocaleLowerCase() + storagePath.substring(1);
+      }
+
+      removeStorage = dir.removeCallback;
     }
-  },
-
-  // Set the CLI version here before activation to ensure we don't accidentally try to download a cli
-  async () => {
-    await workspace.getConfiguration().update('codeQL.cli.executablePath', process.env.CLI_PATH, ConfigurationTarget.Global);
-  },
-
-  // Create the temp directory to be used as extension local storage.
-  () => {
-    const dir = tmp.dirSync();
-    storagePath = fs.realpathSync(dir.name);
-    if (storagePath.substring(0, 2).match(/[A-Z]:/)) {
-      storagePath = storagePath.substring(0, 1).toLocaleLowerCase() + storagePath.substring(1);
-    }
-
-    removeStorage = dir.removeCallback;
-  }]);
-
-
+  ]);
 
   mocha.globalTeardown([
+    // ensure etension is cleaned up.
+    async () => {
+      const extension = await extensions.getExtension<CodeQLExtensionInterface | {}>('GitHub.vscode-codeql')!.activate();
+      // This shuts down the extension and can only be run after all tests have completed.
+      // If this is not called, then the test process will hang.
+      if ('dispose' in extension) {
+        extension.dispose();
+      }
+    },
     // ensure temp directory is cleaned up.
     () => {
       removeStorage?.();

extensions/ql-vscode/src/vscode-tests/cli-integration/queries.test.ts

@@ -13,14 +13,19 @@ import { importArchiveDatabase } from '../../databaseFetcher';
 import { compileAndRunQueryAgainstDatabase } from '../../run-queries';
 import { CodeQLCliServer } from '../../cli';
 import { QueryServerClient } from '../../queryserver-client';
+import { skipIfNoCodeQL } from '../ensureCli';
 
 
 /**
  * Integration tests for queries
  */
-describe.only('Queries', function() {
+describe('Queries', function() {
   this.timeout(20000);
 
+  before(function() {
+    skipIfNoCodeQL(this);
+  });
+
   let dbItem: DatabaseItem;
   let databaseManager: DatabaseManager;
   let cli: CodeQLCliServer;
@@ -67,8 +72,6 @@ describe.only('Queries', function() {
 
   afterEach(() => {
     try {
-      databaseManager?.dispose();
-      qs?.dispose();
       sandbox.restore();
     } catch (e) {
       fail(e);

extensions/ql-vscode/src/vscode-tests/cli-integration/query.test.ts

@@ -8,9 +8,11 @@ import { CancellationTokenSource } from 'vscode-jsonrpc';
 import * as messages from '../../pure/messages';
 import * as qsClient from '../../queryserver-client';
 import * as cli from '../../cli';
-import { ProgressReporter, Logger } from '../../logging';
 import { ColumnValue } from '../../pure/bqrs-cli-types';
-import { FindDistributionResultKind } from '../../distribution';
+import { extensions } from 'vscode';
+import { CodeQLExtensionInterface } from '../../extension';
+import { fail } from 'assert';
+import { skipIfNoCodeQL } from '../ensureCli';
 
 
 const baseDir = path.join(__dirname, '../../../test/data');
@@ -81,65 +83,33 @@ const queryTestCases: QueryTestCase[] = [
 
 describe('using the query server', function() {
   before(function() {
-    if (process.env['CODEQL_PATH'] === undefined) {
-      console.log('The environment variable CODEQL_PATH is not set. The query server tests, which require the CodeQL CLI, will be skipped.');
-      this.skip();
-    }
+    skipIfNoCodeQL(this);
   });
 
   // Note this does not work with arrow functions as the test case bodies:
   // ensure they are all written with standard anonymous functions.
-  this.timeout(10000);
+  this.timeout(20000);
 
-  const codeQlPath = process.env['CODEQL_PATH']!;
   let qs: qsClient.QueryServerClient;
   let cliServer: cli.CodeQLCliServer;
   const queryServerStarted = new Checkpoint<void>();
-  after(() => {
-    qs?.dispose();
-    cliServer?.dispose();
+
+  beforeEach(async () => {
+    try {
+      const extension = await extensions.getExtension<CodeQLExtensionInterface | {}>('GitHub.vscode-codeql')!.activate();
+      if ('cliServer' in extension && 'qs' in extension) {
+        cliServer = extension.cliServer;
+        qs = extension.qs;
+        cliServer.quiet = true;
+      } else {
+        throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
+      }
+    } catch (e) {
+      fail(e);
+    }
   });
 
   it('should be able to start the query server', async function() {
-    const consoleProgressReporter: ProgressReporter = {
-      report: (v: { message: string }) => console.log(`progress reporter says ${v.message}`)
-    };
-    const logger: Logger = {
-      log: async (s: string) => console.log('logger says', s),
-      show: () => { /**/ },
-      removeAdditionalLogLocation: async () => { /**/ },
-      getBaseLocation: () => ''
-    };
-    cliServer = new cli.CodeQLCliServer(
-      {
-        async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
-          return codeQlPath;
-        },
-        getDistribution: async () => {
-          return {
-            kind: FindDistributionResultKind.NoDistribution
-          };
-        }
-      },
-      {
-        numberTestThreads: 2
-      },
-      logger
-    );
-    qs = new qsClient.QueryServerClient(
-      {
-        codeQlPath,
-        numThreads: 1,
-        queryMemoryMb: 1024,
-        timeoutSecs: 1000,
-        debug: false
-      },
-      cliServer,
-      {
-        logger
-      },
-      task => task(consoleProgressReporter, token)
-    );
     await qs.startQueryServer();
     queryServerStarted.resolve();
   });

extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts

@@ -4,7 +4,8 @@ import { SemVer } from 'semver';
 
 import { CodeQLCliServer } from '../../cli';
 import { CodeQLExtensionInterface } from '../../extension';
-
+import { skipIfNoCodeQL } from '../ensureCli';
+import { getOnDiskWorkspaceFolders } from '../../helpers';
 
 /**
  * Perform proper integration tests by running the CLI
@@ -23,10 +24,6 @@ describe('Use cli', function() {
     }
   });
 
-  afterEach(() => {
-    cli.dispose();
-  });
-
   it('should have the correct version of the cli', async () => {
     expect(
       (await cli.getVersion()).toString()
@@ -42,4 +39,15 @@ describe('Use cli', function() {
       '--off-heap-ram=4096'
     ]);
   });
+
+  it.only('should resolve query packs', async function() {
+    skipIfNoCodeQL(this);
+    const qlpacks = await cli.resolveQlpacks(getOnDiskWorkspaceFolders());
+    // should have a bunch of qlpacks. just check that a few known ones exist
+    expect(qlpacks['codeql-cpp']).not.to.be.undefined;
+    expect(qlpacks['codeql-csharp']).not.to.be.undefined;
+    expect(qlpacks['codeql-java']).not.to.be.undefined;
+    expect(qlpacks['codeql-javascript']).not.to.be.undefined;
+    expect(qlpacks['codeql-python']).not.to.be.undefined;
+  });
 });

extensions/ql-vscode/src/vscode-tests/ensureCli.ts

@@ -2,6 +2,7 @@ import * as fs from 'fs-extra';
 import * as path from 'path';
 import { DistributionManager, extractZipArchive, codeQlLauncherName } from '../distribution';
 import fetch from 'node-fetch';
+import { workspace } from 'vscode';
 
 /**
  * This module ensures that the proper CLI is available for tests of the extension.
@@ -105,6 +106,22 @@ export async function ensureCli(useCli: boolean) {
   }
 }
 
+/**
+ * Heuristically determines if the codeql libraries are installed in this
+ * workspace. Looks for the existance of a folder whose path ends in `/codeql`
+ */
+function hasCodeQL() {
+  const folders = workspace.workspaceFolders;
+  return !!folders?.some(folder => folder.uri.path.endsWith('/codeql'));
+}
+
+export function skipIfNoCodeQL(context: Mocha.Context) {
+  if (!hasCodeQL()) {
+    console.log('The CodeQL libraries are not available as a folder in this workspace. To fix: checkout the github/codeql repository and set the TEST_CODEQL_PATH environment variable to the checked out directory.');
+    context.skip();
+  }
+}
+
 /**
  * Url to download from
  */
@@ -116,7 +133,7 @@ function getCliDownloadUrl(assetName: string) {
  * Directory to place the downloaded cli into
  */
 function getDownloadFilePath(assetName: string) {
-  const dir = path.join(CLI_BASE_DIR, 'assets');
+  const dir = path.join(CLI_BASE_DIR, 'assets', CLI_VERSION);
   fs.mkdirpSync(dir);
   return path.join(dir, assetName);
 }

extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts

@@ -122,9 +122,10 @@ function getLaunchArgs(dir: TestDir) {
       ];
 
     case TestDir.CliIntegration:
+      // CLI integration tests requires a multi-root workspace so that the data and the QL sources are accessible.
       return [
         path.resolve(__dirname, '../../test/data'),
-        process.env.QL_PATH!
+        process.env.TEST_CODEQL_PATH!
       ];
 
     default: