Merge pull request #1746 from github/koesie10/remote-queries-tests-tar

Change remote queries to test against submitted data

extensions/ql-vscode/package-lock.json

@@ -96,6 +96,7 @@
         "@types/sinon-chai": "~3.2.3",
         "@types/stream-chain": "~2.0.1",
         "@types/stream-json": "~1.7.1",
+        "@types/tar-stream": "^2.2.2",
         "@types/through2": "^2.0.36",
         "@types/tmp": "^0.1.0",
         "@types/unzipper": "~0.10.1",
@@ -136,6 +137,7 @@
         "proxyquire": "~2.1.3",
         "sinon": "~14.0.0",
         "sinon-chai": "~3.5.0",
+        "tar-stream": "^2.2.0",
         "through2": "^4.0.2",
         "ts-jest": "^29.0.1",
         "ts-json-schema-generator": "^1.1.2",
@@ -13517,6 +13519,15 @@
       "integrity": "sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ==",
       "dev": true
     },
+    "node_modules/@types/tar-stream": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/@types/tar-stream/-/tar-stream-2.2.2.tgz",
+      "integrity": "sha512-1AX+Yt3icFuU6kxwmPakaiGrJUwG44MpuiqPg4dSolRFk6jmvs4b3IbUol9wKDLIgU76gevn3EwE8y/DkSJCZQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/testing-library__jest-dom": {
       "version": "5.14.5",
       "resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.5.tgz",
@@ -50622,6 +50633,15 @@
       "integrity": "sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ==",
       "dev": true
     },
+    "@types/tar-stream": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/@types/tar-stream/-/tar-stream-2.2.2.tgz",
+      "integrity": "sha512-1AX+Yt3icFuU6kxwmPakaiGrJUwG44MpuiqPg4dSolRFk6jmvs4b3IbUol9wKDLIgU76gevn3EwE8y/DkSJCZQ==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/testing-library__jest-dom": {
       "version": "5.14.5",
       "resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.5.tgz",

extensions/ql-vscode/package.json

@@ -1380,6 +1380,7 @@
     "@types/sinon-chai": "~3.2.3",
     "@types/stream-chain": "~2.0.1",
     "@types/stream-json": "~1.7.1",
+    "@types/tar-stream": "^2.2.2",
     "@types/through2": "^2.0.36",
     "@types/tmp": "^0.1.0",
     "@types/unzipper": "~0.10.1",
@@ -1420,6 +1421,7 @@
     "proxyquire": "~2.1.3",
     "sinon": "~14.0.0",
     "sinon-chai": "~3.5.0",
+    "tar-stream": "^2.2.0",
     "through2": "^4.0.2",
     "ts-jest": "^29.0.1",
     "ts-json-schema-generator": "^1.1.2",

extensions/ql-vscode/src/remote-queries/gh-api/gh-api-client.ts

@@ -1,5 +1,6 @@
 import { Credentials } from '../../authentication';
 import { OctokitResponse } from '@octokit/types/dist-types';
+import { RemoteQueriesSubmission } from '../shared/remote-queries';
 import { VariantAnalysisSubmission } from '../shared/variant-analysis';
 import {
   VariantAnalysis,
@@ -7,6 +8,7 @@ import {
   VariantAnalysisSubmissionRequest
 } from './variant-analysis';
 import { Repository } from './repository';
+import { RemoteQueriesResponse, RemoteQueriesSubmissionRequest } from './remote-queries';
 
 export async function submitVariantAnalysis(
   credentials: Credentials,
@@ -116,3 +118,31 @@ export async function createGist(
   }
   return response.data.html_url;
 }
+
+export async function submitRemoteQueries(
+  credentials: Credentials,
+  submissionDetails: RemoteQueriesSubmission
+): Promise<RemoteQueriesResponse> {
+  const octokit = await credentials.getOctokit();
+
+  const { ref, language, repositories, repositoryLists, repositoryOwners, queryPack, controllerRepoId } = submissionDetails;
+
+  const data: RemoteQueriesSubmissionRequest = {
+    ref,
+    language,
+    repositories,
+    repository_lists: repositoryLists,
+    repository_owners: repositoryOwners,
+    query_pack: queryPack,
+  };
+
+  const response: OctokitResponse<RemoteQueriesResponse> = await octokit.request(
+    'POST /repositories/:controllerRepoId/code-scanning/codeql/queries',
+    {
+      controllerRepoId,
+      data
+    }
+  );
+
+  return response.data;
+}

extensions/ql-vscode/src/remote-queries/gh-api/remote-queries.ts

@@ -0,0 +1,20 @@
+export interface RemoteQueriesSubmissionRequest {
+  ref: string;
+  language: string;
+  repositories?: string[];
+  repository_lists?: string[];
+  repository_owners?: string[];
+  query_pack: string;
+}
+
+export interface RemoteQueriesResponse {
+  workflow_run_id: number,
+  errors?: {
+    invalid_repositories?: string[],
+    repositories_without_database?: string[],
+    private_repositories?: string[],
+    cutoff_repositories?: string[],
+    cutoff_repositories_count?: number,
+  },
+  repositories_queried: string[],
+}

extensions/ql-vscode/src/remote-queries/run-remote-query.ts

@@ -18,13 +18,14 @@ import * as cli from '../cli';
 import { logger } from '../logging';
 import { getActionBranch, getRemoteControllerRepo, isVariantAnalysisLiveResultsEnabled, setRemoteControllerRepo } from '../config';
 import { ProgressCallback, UserCancellationException } from '../commandRunner';
-import { OctokitResponse, RequestError } from '@octokit/types/dist-types';
+import { RequestError } from '@octokit/types/dist-types';
 import { RemoteQuery } from './remote-query';
 import { RemoteQuerySubmissionResult } from './remote-query-submission-result';
 import { QueryMetadata } from '../pure/interface-types';
 import { getErrorMessage, REPO_REGEX } from '../pure/helpers-pure';
 import { pluralize } from '../pure/word';
 import * as ghApiClient from './gh-api/gh-api-client';
+import { RemoteQueriesResponse } from './gh-api/remote-queries';
 import { getRepositorySelection, isValidSelection, RepositorySelection } from './repository-selection';
 import { parseVariantAnalysisQueryLanguage, VariantAnalysisSubmission } from './shared/variant-analysis';
 import { Repository } from './shared/repository';
@@ -39,18 +40,6 @@ export interface QlPack {
   defaultSuiteFile?: string;
 }
 
-interface QueriesResponse {
-  workflow_run_id: number,
-  errors?: {
-    invalid_repositories?: string[],
-    repositories_without_database?: string[],
-    private_repositories?: string[],
-    cutoff_repositories?: string[],
-    cutoff_repositories_count?: number,
-  },
-  repositories_queried: string[],
-}
-
 /**
  * Well-known names for the query pack used by the server.
  */
@@ -172,7 +161,7 @@ function isFileSystemRoot(dir: string): boolean {
   return pathObj.root === dir && pathObj.base === '';
 }
 
-async function createRemoteQueriesTempDirectory() {
+export async function createRemoteQueriesTempDirectory() {
   const remoteQueryDir = await tmp.dir({ dir: tmpDir.name, unsafeCleanup: true });
   const queryPackDir = path.join(remoteQueryDir.path, 'query-pack');
   await fs.mkdirp(queryPackDir);
@@ -389,7 +378,7 @@ async function runRemoteQueriesApiRequest(
   controllerRepo: Repository,
   queryPackBase64: string,
   dryRun = false
-): Promise<void | QueriesResponse> {
+): Promise<void | RemoteQueriesResponse> {
   const data = {
     ref,
     language,
@@ -412,17 +401,18 @@ async function runRemoteQueriesApiRequest(
   }
 
   try {
-    const octokit = await credentials.getOctokit();
+    const response = await ghApiClient.submitRemoteQueries(credentials, {
-    const response: OctokitResponse<QueriesResponse, number> = await octokit.request(
+      ref,
-      'POST /repositories/:controllerRepoId/code-scanning/codeql/queries',
+      language,
-      {
+      repositories: repoSelection.repositories,
-        controllerRepoId: controllerRepo.id,
+      repositoryLists: repoSelection.repositoryLists,
-        data
+      repositoryOwners: repoSelection.owners,
-      }
+      queryPack: queryPackBase64,
-    );
+      controllerRepoId: controllerRepo.id,
-    const { popupMessage, logMessage } = parseResponse(controllerRepo, response.data);
+    });
+    const { popupMessage, logMessage } = parseResponse(controllerRepo, response);
     void showAndLogInformationMessage(popupMessage, { fullMessage: logMessage });
-    return response.data;
+    return response;
   } catch (error: any) {
     if (error.status === 404) {
       void showAndLogErrorMessage(`Controller repository was not found. Please make sure it's a valid repo name.${eol}`);
@@ -436,7 +426,7 @@ const eol = os.EOL;
 const eol2 = os.EOL + os.EOL;
 
 // exported for testing only
-export function parseResponse(controllerRepo: Repository, response: QueriesResponse) {
+export function parseResponse(controllerRepo: Repository, response: RemoteQueriesResponse) {
   const repositoriesQueried = response.repositories_queried;
   const repositoryCount = repositoriesQueried.length;
 

extensions/ql-vscode/src/remote-queries/shared/remote-queries.ts

@@ -0,0 +1,10 @@
+export interface RemoteQueriesSubmission {
+  ref: string;
+  language: string;
+  repositories?: string[];
+  repositoryLists?: string[];
+  repositoryOwners?: string[];
+  queryPack: string;
+
+  controllerRepoId: number;
+}

extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/run-remote-query.test.ts

@@ -2,7 +2,6 @@ import { assert, expect } from 'chai';
 import * as path from 'path';
 import * as sinon from 'sinon';
 import { CancellationTokenSource, ExtensionContext, extensions, QuickPickItem, Uri, window } from 'vscode';
-import * as fs from 'fs-extra';
 import * as os from 'os';
 import * as yaml from 'js-yaml';
 
@@ -25,6 +24,8 @@ import { createMockExtensionContext } from '../../no-workspace';
 import { VariantAnalysisManager } from '../../../remote-queries/variant-analysis-manager';
 import { OutputChannelLogger } from '../../../logging';
 import { VariantAnalysisResultsManager } from '../../../remote-queries/variant-analysis-results-manager';
+import { RemoteQueriesSubmission } from '../../../remote-queries/shared/remote-queries';
+import { readBundledPack } from '../../utils/bundled-pack-helpers';
 
 describe('Remote queries', function() {
   const baseDir = path.join(__dirname, '../../../../src/vscode-tests/cli-integration');
@@ -97,12 +98,26 @@ describe('Remote queries', function() {
   });
 
   describe('when live results are not enabled', () => {
+    let mockSubmitRemoteQueries: sinon.SinonStub;
+
+    beforeEach(() => {
+      mockSubmitRemoteQueries = sandbox.stub(ghApiClient, 'submitRemoteQueries').resolves({
+        workflow_run_id: 20,
+        repositories_queried: ['octodemo/hello-world-1'],
+      });
+    });
+
     it('should run a remote query that is part of a qlpack', async () => {
       const fileUri = getFile('data-remote-qlpack/in-pack.ql');
 
-      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
+      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
       expect(querySubmissionResult).to.be.ok;
-      const queryPackRootDir = querySubmissionResult!.queryDirPath!;
+
+      expect(mockSubmitRemoteQueries).to.have.been.calledOnce;
+
+      const request: RemoteQueriesSubmission = mockSubmitRemoteQueries.getCall(0).lastArg;
+
+      const packFS = await readBundledPack(request.queryPack);
 
       // to retrieve the list of repositories
       expect(showQuickPickSpy).to.have.been.calledOnce;
@@ -110,43 +125,25 @@ describe('Remote queries', function() {
       expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
 
       // check a few files that we know should exist and others that we know should not
-
+      expect(packFS.fileExists('in-pack.ql')).to.be.true;
-      // the tarball to deliver to the server
+      expect(packFS.fileExists('lib.qll')).to.be.true;
-      expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
+      expect(packFS.fileExists('qlpack.yml')).to.be.true;
-
-      const queryPackDir = path.join(queryPackRootDir, 'query-pack');
-
-      expect(fs.existsSync(path.join(queryPackDir, 'in-pack.ql'))).to.be.true;
-      expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.true;
-      expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
 
       // depending on the cli version, we should have one of these files
       expect(
-        fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
+        packFS.fileExists('qlpack.lock.yml') ||
-        fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
+        packFS.fileExists('codeql-pack.lock.yml')
       ).to.be.true;
-      expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
+      expect(packFS.fileExists('not-in-pack.ql')).to.be.false;
 
-      // the compiled pack
-      const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
-
-      expect(fs.existsSync(path.join(compiledPackDir, 'in-pack.ql'))).to.be.true;
-      expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.true;
-      expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
       // should have generated a correct qlpack file
-      const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
+      const qlpackContents: any = yaml.load(packFS.fileContents('qlpack.yml').toString('utf-8'));
       expect(qlpackContents.name).to.equal('codeql-remote/query');
 
-      // depending on the cli version, we should have one of these files
+      verifyQlPack('in-pack.ql', packFS.fileContents('qlpack.yml'), '0.0.0', await pathSerializationBroken());
-      expect(
-        fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
-        fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
-      ).to.be.true;
-      expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
-      verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'in-pack.ql', '0.0.0', await pathSerializationBroken());
 
-      const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
+      const libraryDir = '.codeql/libraries/codeql';
-      const packNames = fs.readdirSync(libraryDir).sort();
+      const packNames = packFS.directoryContents(libraryDir).sort();
 
       // check dependencies.
       // 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
@@ -158,9 +155,14 @@ describe('Remote queries', function() {
     it('should run a remote query that is not part of a qlpack', async () => {
       const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
 
-      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
+      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
       expect(querySubmissionResult).to.be.ok;
-      const queryPackRootDir = querySubmissionResult!.queryDirPath!;
+
+      expect(mockSubmitRemoteQueries).to.have.been.calledOnce;
+
+      const request: RemoteQueriesSubmission = mockSubmitRemoteQueries.getCall(0).lastArg;
+
+      const packFS = await readBundledPack(request.queryPack);
 
       // to retrieve the list of repositories
       // and a second time to ask for the language
@@ -169,43 +171,27 @@ describe('Remote queries', function() {
       expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
 
       // check a few files that we know should exist and others that we know should not
-
+      expect(packFS.fileExists('in-pack.ql')).to.be.true;
-      // the tarball to deliver to the server
+      expect(packFS.fileExists('qlpack.yml')).to.be.true;
-      expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
-
-      const queryPackDir = path.join(queryPackRootDir, 'query-pack');
-
-      expect(fs.existsSync(path.join(queryPackDir, 'in-pack.ql'))).to.be.true;
-      expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
       // depending on the cli version, we should have one of these files
       expect(
-        fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
+        packFS.fileExists('qlpack.lock.yml') ||
-        fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
+        packFS.fileExists('codeql-pack.lock.yml')
       ).to.be.true;
-      expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.false;
+      expect(packFS.fileExists('lib.qll')).to.be.false;
-      expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
+      expect(packFS.fileExists('not-in-pack.ql')).to.be.false;
 
       // the compiled pack
-      const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
+      verifyQlPack('in-pack.ql', packFS.fileContents('qlpack.yml'), '0.0.0', await pathSerializationBroken());
-      expect(fs.existsSync(path.join(compiledPackDir, 'in-pack.ql'))).to.be.true;
-      expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
-      verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'in-pack.ql', '0.0.0', await pathSerializationBroken());
 
-      // depending on the cli version, we should have one of these files
-      expect(
-        fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
-        fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
-      ).to.be.true;
-      expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.false;
-      expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
       // should have generated a correct qlpack file
-      const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
+      const qlpackContents: any = yaml.load(packFS.fileContents('qlpack.yml').toString('utf-8'));
       expect(qlpackContents.name).to.equal('codeql-remote/query');
       expect(qlpackContents.version).to.equal('0.0.0');
       expect(qlpackContents.dependencies?.['codeql/javascript-all']).to.equal('*');
 
-      const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
+      const libraryDir = '.codeql/libraries/codeql';
-      const packNames = fs.readdirSync(libraryDir).sort();
+      const packNames = packFS.directoryContents(libraryDir).sort();
 
       // check dependencies.
       // 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
@@ -217,9 +203,14 @@ describe('Remote queries', function() {
     it('should run a remote query that is nested inside a qlpack', async () => {
       const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql');
 
-      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
+      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
       expect(querySubmissionResult).to.be.ok;
-      const queryPackRootDir = querySubmissionResult!.queryDirPath!;
+
+      expect(mockSubmitRemoteQueries).to.have.been.calledOnce;
+
+      const request: RemoteQueriesSubmission = mockSubmitRemoteQueries.getCall(0).lastArg;
+
+      const packFS = await readBundledPack(request.queryPack);
 
       // to retrieve the list of repositories
       expect(showQuickPickSpy).to.have.been.calledOnce;
@@ -227,43 +218,27 @@ describe('Remote queries', function() {
       expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
 
       // check a few files that we know should exist and others that we know should not
-
+      expect(packFS.fileExists('subfolder/in-pack.ql')).to.be.true;
-      // the tarball to deliver to the server
+      expect(packFS.fileExists('qlpack.yml')).to.be.true;
-      expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
-
-      const queryPackDir = path.join(queryPackRootDir, 'query-pack');
-
-      expect(fs.existsSync(path.join(queryPackDir, 'subfolder/in-pack.ql'))).to.be.true;
-      expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
       // depending on the cli version, we should have one of these files
       expect(
-        fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
+        packFS.fileExists('qlpack.lock.yml') ||
-        fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
+        packFS.fileExists('codeql-pack.lock.yml')
       ).to.be.true;
-      expect(fs.existsSync(path.join(queryPackDir, 'otherfolder/lib.qll'))).to.be.true;
+      expect(packFS.fileExists('otherfolder/lib.qll')).to.be.true;
-      expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
+      expect(packFS.fileExists('not-in-pack.ql')).to.be.false;
 
       // the compiled pack
-      const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
+      verifyQlPack('subfolder/in-pack.ql', packFS.fileContents('qlpack.yml'), '0.0.0', await pathSerializationBroken());
-      expect(fs.existsSync(path.join(compiledPackDir, 'otherfolder/lib.qll'))).to.be.true;
-      expect(fs.existsSync(path.join(compiledPackDir, 'subfolder/in-pack.ql'))).to.be.true;
-      expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
-      verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'subfolder/in-pack.ql', '0.0.0', await pathSerializationBroken());
 
-      // depending on the cli version, we should have one of these files
-      expect(
-        fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
-        fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
-      ).to.be.true;
-      expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
       // should have generated a correct qlpack file
-      const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
+      const qlpackContents: any = yaml.load(packFS.fileContents('qlpack.yml').toString('utf-8'));
       expect(qlpackContents.name).to.equal('codeql-remote/query');
       expect(qlpackContents.version).to.equal('0.0.0');
       expect(qlpackContents.dependencies?.['codeql/javascript-all']).to.equal('*');
 
-      const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
+      const libraryDir = '.codeql/libraries/codeql';
-      const packNames = fs.readdirSync(libraryDir).sort();
+      const packNames = packFS.directoryContents(libraryDir).sort();
 
       // check dependencies.
       // 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
@@ -275,7 +250,7 @@ describe('Remote queries', function() {
     it('should cancel a run before uploading', async () => {
       const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
 
-      const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
+      const promise = runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
 
       cancellationTokenSource.cancel();
 
@@ -301,7 +276,7 @@ describe('Remote queries', function() {
     it('should run a variant analysis that is part of a qlpack', async () => {
       const fileUri = getFile('data-remote-qlpack/in-pack.ql');
 
-      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
+      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
       expect(querySubmissionResult).to.be.ok;
       const variantAnalysis = querySubmissionResult!.variantAnalysis!;
       expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -314,7 +289,7 @@ describe('Remote queries', function() {
     it('should run a remote query that is not part of a qlpack', async () => {
       const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
 
-      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
+      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
       expect(querySubmissionResult).to.be.ok;
       const variantAnalysis = querySubmissionResult!.variantAnalysis!;
       expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -327,7 +302,7 @@ describe('Remote queries', function() {
     it('should run a remote query that is nested inside a qlpack', async () => {
       const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql');
 
-      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
+      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
       expect(querySubmissionResult).to.be.ok;
       const variantAnalysis = querySubmissionResult!.variantAnalysis!;
       expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -340,7 +315,7 @@ describe('Remote queries', function() {
     it('should cancel a run before uploading', async () => {
       const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
 
-      const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
+      const promise = runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
 
       cancellationTokenSource.cancel();
 
@@ -353,8 +328,8 @@ describe('Remote queries', function() {
     });
   });
 
-  function verifyQlPack(qlpackPath: string, queryPath: string, packVersion: string, pathSerializationBroken: boolean) {
+  function verifyQlPack(queryPath: string, contents: Buffer, packVersion: string, pathSerializationBroken: boolean) {
-    const qlPack = yaml.load(fs.readFileSync(qlpackPath, 'utf8')) as QlPack;
+    const qlPack = yaml.load(contents.toString('utf-8')) as QlPack;
 
     if (pathSerializationBroken) {
       // the path serialization is broken, so we force it to be the path in the pack to be same as the query path

extensions/ql-vscode/src/vscode-tests/utils/bundled-pack-helpers.ts

@@ -0,0 +1,76 @@
+import { Readable } from 'stream';
+import * as tar from 'tar-stream';
+import { Headers } from 'tar-stream';
+import { pipeline } from 'stream/promises';
+import { createGunzip } from 'zlib';
+
+export interface QueryPackFS {
+  fileExists: (name: string) => boolean;
+  fileContents: (name: string) => Buffer;
+  directoryContents: (name: string) => string[];
+}
+
+export async function readBundledPack(base64Pack: string): Promise<QueryPackFS> {
+  const buffer = Buffer.from(base64Pack, 'base64');
+  const stream = Readable.from(buffer);
+
+  const extract = tar.extract();
+
+  const files: Record<string, {
+    headers: Headers,
+    contents: Buffer,
+  }> = {};
+
+  extract.on('entry', function(headers: Headers, stream, next) {
+    const buffers: Buffer[] = [];
+
+    stream.on('data', chunk => buffers.push(chunk));
+    stream.on('end', () => {
+      files[headers.name] = {
+        headers,
+        contents: Buffer.concat(buffers),
+      };
+
+      next();
+    });
+    stream.on('error', err => {
+      console.error(err);
+      next();
+    });
+  });
+
+  await pipeline(stream, createGunzip(), extract);
+
+  const directories: Record<string, number> = {};
+  for (let file of Object.keys(files)) {
+    while (file.indexOf('/') > 0) {
+      const directory = file.substring(0, file.lastIndexOf('/'));
+      if (!(directory in directories)) {
+        directories[directory] = 0;
+      }
+
+      directories[directory]++;
+
+      file = directory;
+    }
+  }
+
+  return {
+    fileExists: (name: string) => {
+      return files[name]?.headers.type === 'file';
+    },
+    fileContents: (name: string): Buffer => {
+      const file = files[name];
+      if (file?.headers.type === 'file') {
+        return file.contents;
+      }
+
+      throw new Error(`File ${name} does not exist`);
+    },
+    directoryContents: (name: string): string[] => {
+      return Object.keys(directories)
+        .filter(dir => dir.startsWith(name) && dir !== name && dir.substring(name.length + 1).split('/').length === 1)
+        .map(dir => dir.substring(name.length + 1));
+    },
+  };
+}