mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Change remote queries to test against submitted data

Browse Source 
The remote queries tests were testing the data on the filesystem, rather
than the data submitted to the server. This required using a `dryRun`
parameter to prevent deleting the temporary directory, while we can
actually just test against the submitted data.

This will create an in-memory filesystem of the submitted query pack by
un-tar-gz'ing the query pack into memory and using that to test the
existence of certain files.
This commit is contained in:
Koen Vlaswinkel
2022-11-08 16:05:38 +01:00
parent 5905cf8811
commit 2dad33f2ba
8 changed files with 241 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
extensions/ql-vscode/package-lock.json generated
View File

@@ -96,6 +96,7 @@
"@types/sinon-chai": "~3.2.3",
"@types/stream-chain": "~2.0.1",
"@types/stream-json": "~1.7.1",
"@types/tar-stream": "^2.2.2",
"@types/through2": "^2.0.36",
"@types/tmp": "^0.1.0",
"@types/unzipper": "~0.10.1",
@@ -13517,6 +13518,15 @@
"integrity": "sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ==",
"dev": true
},
"node_modules/@types/tar-stream": {
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/@types/tar-stream/-/tar-stream-2.2.2.tgz",
"integrity": "sha512-1AX+Yt3icFuU6kxwmPakaiGrJUwG44MpuiqPg4dSolRFk6jmvs4b3IbUol9wKDLIgU76gevn3EwE8y/DkSJCZQ==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/testing-library__jest-dom": {
"version": "5.14.5",
"resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.5.tgz",
@@ -50622,6 +50632,15 @@
"integrity": "sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ==",
"dev": true
},
"@types/tar-stream": {
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/@types/tar-stream/-/tar-stream-2.2.2.tgz",
"integrity": "sha512-1AX+Yt3icFuU6kxwmPakaiGrJUwG44MpuiqPg4dSolRFk6jmvs4b3IbUol9wKDLIgU76gevn3EwE8y/DkSJCZQ==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/testing-library__jest-dom": {
"version": "5.14.5",
"resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.5.tgz",

1
extensions/ql-vscode/package.json
View File

@@ -1380,6 +1380,7 @@
"@types/sinon-chai": "~3.2.3",
"@types/stream-chain": "~2.0.1",
"@types/stream-json": "~1.7.1",
"@types/tar-stream": "^2.2.2",
"@types/through2": "^2.0.36",
"@types/tmp": "^0.1.0",
"@types/unzipper": "~0.10.1",

30
extensions/ql-vscode/src/remote-queries/gh-api/gh-api-client.ts
View File

@@ -1,5 +1,6 @@
import { Credentials } from '../../authentication';
import { OctokitResponse } from '@octokit/types/dist-types';
import { RemoteQueriesSubmission } from '../shared/remote-queries';
import { VariantAnalysisSubmission } from '../shared/variant-analysis';
import {
VariantAnalysis,
@@ -7,6 +8,7 @@ import {
VariantAnalysisSubmissionRequest
} from './variant-analysis';
import { Repository } from './repository';
import { RemoteQueriesResponse, RemoteQueriesSubmissionRequest } from './remote-queries';
export async function submitVariantAnalysis(
credentials: Credentials,
@@ -116,3 +118,31 @@ export async function createGist(
}
return response.data.html_url;
}
export async function submitRemoteQueries(
credentials: Credentials,
submissionDetails: RemoteQueriesSubmission
): Promise<RemoteQueriesResponse> {
const octokit = await credentials.getOctokit();
const { ref, language, repositories, repositoryLists, repositoryOwners, queryPack, controllerRepoId } = submissionDetails;
const data: RemoteQueriesSubmissionRequest = {
ref,
language,
repositories,
repository_lists: repositoryLists,
repository_owners: repositoryOwners,
query_pack: queryPack,
};
const response: OctokitResponse<RemoteQueriesResponse> = await octokit.request(
'POST /repositories/:controllerRepoId/code-scanning/codeql/queries',
{
controllerRepoId,
data
}
);
return response.data;
}

20
extensions/ql-vscode/src/remote-queries/gh-api/remote-queries.ts Normal file
View File

@@ -0,0 +1,20 @@
export interface RemoteQueriesSubmissionRequest {
ref: string;
language: string;
repositories?: string[];
repository_lists?: string[];
repository_owners?: string[];
query_pack: string;
}
export interface RemoteQueriesResponse {
workflow_run_id: number,
errors?: {
invalid_repositories?: string[],
repositories_without_database?: string[],
private_repositories?: string[],
cutoff_repositories?: string[],
cutoff_repositories_count?: number,
},
repositories_queried: string[],
}

42
extensions/ql-vscode/src/remote-queries/run-remote-query.ts
View File

@@ -18,13 +18,14 @@ import * as cli from '../cli';
import { logger } from '../logging';
import { getActionBranch, getRemoteControllerRepo, isVariantAnalysisLiveResultsEnabled, setRemoteControllerRepo } from '../config';
import { ProgressCallback, UserCancellationException } from '../commandRunner';
import { OctokitResponse, RequestError } from '@octokit/types/dist-types';
import { RequestError } from '@octokit/types/dist-types';
import { RemoteQuery } from './remote-query';
import { RemoteQuerySubmissionResult } from './remote-query-submission-result';
import { QueryMetadata } from '../pure/interface-types';
import { getErrorMessage, REPO_REGEX } from '../pure/helpers-pure';
import { pluralize } from '../pure/word';
import * as ghApiClient from './gh-api/gh-api-client';
import { RemoteQueriesResponse } from './gh-api/remote-queries';
import { getRepositorySelection, isValidSelection, RepositorySelection } from './repository-selection';
import { parseVariantAnalysisQueryLanguage, VariantAnalysisSubmission } from './shared/variant-analysis';
import { Repository } from './shared/repository';
@@ -39,18 +40,6 @@ export interface QlPack {
defaultSuiteFile?: string;
}
interface QueriesResponse {
workflow_run_id: number,
errors?: {
invalid_repositories?: string[],
repositories_without_database?: string[],
private_repositories?: string[],
cutoff_repositories?: string[],
cutoff_repositories_count?: number,
},
repositories_queried: string[],
}
/**
* Well-known names for the query pack used by the server.
*/
@@ -172,7 +161,7 @@ function isFileSystemRoot(dir: string): boolean {
return pathObj.root === dir && pathObj.base === '';
}
async function createRemoteQueriesTempDirectory() {
export async function createRemoteQueriesTempDirectory() {
const remoteQueryDir = await tmp.dir({ dir: tmpDir.name, unsafeCleanup: true });
const queryPackDir = path.join(remoteQueryDir.path, 'query-pack');
await fs.mkdirp(queryPackDir);
@@ -378,7 +367,7 @@ async function runRemoteQueriesApiRequest(
controllerRepo: Repository,
queryPackBase64: string,
dryRun = false
): Promise<void | QueriesResponse> {
): Promise<void | RemoteQueriesResponse> {
const data = {
ref,
language,
@@ -401,17 +390,18 @@ async function runRemoteQueriesApiRequest(
}
try {
const octokit = await credentials.getOctokit();
const response: OctokitResponse<QueriesResponse, number> = await octokit.request(
'POST /repositories/:controllerRepoId/code-scanning/codeql/queries',
{
controllerRepoId: controllerRepo.id,
data
}
);
const { popupMessage, logMessage } = parseResponse(controllerRepo, response.data);
const response = await ghApiClient.submitRemoteQueries(credentials, {
ref,
language,
repositories: repoSelection.repositories,
repositoryLists: repoSelection.repositoryLists,
repositoryOwners: repoSelection.owners,
queryPack: queryPackBase64,
controllerRepoId: controllerRepo.id,
});
const { popupMessage, logMessage } = parseResponse(controllerRepo, response);
void showAndLogInformationMessage(popupMessage, { fullMessage: logMessage });
return response.data;
return response;
} catch (error: any) {
if (error.status === 404) {
void showAndLogErrorMessage(`Controller repository was not found. Please make sure it's a valid repo name.${eol}`);
@@ -425,7 +415,7 @@ const eol = os.EOL;
const eol2 = os.EOL + os.EOL;
// exported for testing only
export function parseResponse(controllerRepo: Repository, response: QueriesResponse) {
export function parseResponse(controllerRepo: Repository, response: RemoteQueriesResponse) {
const repositoriesQueried = response.repositories_queried;
const repositoryCount = repositoriesQueried.length;

10
extensions/ql-vscode/src/remote-queries/shared/remote-queries.ts Normal file
View File

@@ -0,0 +1,10 @@
export interface RemoteQueriesSubmission {
ref: string;
language: string;
repositories?: string[];
repositoryLists?: string[];
repositoryOwners?: string[];
queryPack: string;
controllerRepoId: number;
}

163
extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/run-remote-query.test.ts
View File

@@ -2,7 +2,6 @@ import { assert, expect } from 'chai';
import * as path from 'path';
import * as sinon from 'sinon';
import { CancellationTokenSource, ExtensionContext, extensions, QuickPickItem, Uri, window } from 'vscode';
import * as fs from 'fs-extra';
import * as os from 'os';
import * as yaml from 'js-yaml';
@@ -25,6 +24,8 @@ import { createMockExtensionContext } from '../../no-workspace';
import { VariantAnalysisManager } from '../../../remote-queries/variant-analysis-manager';
import { OutputChannelLogger } from '../../../logging';
import { VariantAnalysisResultsManager } from '../../../remote-queries/variant-analysis-results-manager';
import { RemoteQueriesSubmission } from '../../../remote-queries/shared/remote-queries';
import { readBundledPack } from '../../utils/bundled-pack-helpers';
describe('Remote queries', function() {
const baseDir = path.join(__dirname, '../../../../src/vscode-tests/cli-integration');
@@ -97,12 +98,26 @@ describe('Remote queries', function() {
});
describe('when live results are not enabled', () => {
let mockSubmitRemoteQueries: sinon.SinonStub;
beforeEach(() => {
mockSubmitRemoteQueries = sandbox.stub(ghApiClient, 'submitRemoteQueries').resolves({
workflow_run_id: 20,
repositories_queried: ['octodemo/hello-world-1'],
});
});
it('should run a remote query that is part of a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
expect(mockSubmitRemoteQueries).to.have.been.calledOnce;
const request: RemoteQueriesSubmission = mockSubmitRemoteQueries.getCall(0).lastArg;
const packFS = await readBundledPack(request.queryPack);
// to retrieve the list of repositories
expect(showQuickPickSpy).to.have.been.calledOnce;
@@ -110,43 +125,25 @@ describe('Remote queries', function() {
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
// check a few files that we know should exist and others that we know should not
// the tarball to deliver to the server
expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
const queryPackDir = path.join(queryPackRootDir, 'query-pack');
expect(fs.existsSync(path.join(queryPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
expect(packFS.fileExists('in-pack.ql')).to.be.true;
expect(packFS.fileExists('lib.qll')).to.be.true;
expect(packFS.fileExists('qlpack.yml')).to.be.true;
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
packFS.fileExists('qlpack.lock.yml') ||
packFS.fileExists('codeql-pack.lock.yml')
).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
expect(packFS.fileExists('not-in-pack.ql')).to.be.false;
// the compiled pack
const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
expect(fs.existsSync(path.join(compiledPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
// should have generated a correct qlpack file
const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
const qlpackContents: any = yaml.load(packFS.fileContents('qlpack.yml').toString('utf-8'));
expect(qlpackContents.name).to.equal('codeql-remote/query');
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'in-pack.ql', '0.0.0', await pathSerializationBroken());
verifyQlPack('in-pack.ql', packFS.fileContents('qlpack.yml'), '0.0.0', await pathSerializationBroken());
const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
const packNames = fs.readdirSync(libraryDir).sort();
const libraryDir = '.codeql/libraries/codeql';
const packNames = packFS.directoryContents(libraryDir).sort();
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
@@ -158,9 +155,14 @@ describe('Remote queries', function() {
it('should run a remote query that is not part of a qlpack', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
expect(mockSubmitRemoteQueries).to.have.been.calledOnce;
const request: RemoteQueriesSubmission = mockSubmitRemoteQueries.getCall(0).lastArg;
const packFS = await readBundledPack(request.queryPack);
// to retrieve the list of repositories
// and a second time to ask for the language
@@ -169,43 +171,27 @@ describe('Remote queries', function() {
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
// check a few files that we know should exist and others that we know should not
// the tarball to deliver to the server
expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
const queryPackDir = path.join(queryPackRootDir, 'query-pack');
expect(fs.existsSync(path.join(queryPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
expect(packFS.fileExists('in-pack.ql')).to.be.true;
expect(packFS.fileExists('qlpack.yml')).to.be.true;
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
packFS.fileExists('qlpack.lock.yml') ||
packFS.fileExists('codeql-pack.lock.yml')
).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.false;
expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
expect(packFS.fileExists('lib.qll')).to.be.false;
expect(packFS.fileExists('not-in-pack.ql')).to.be.false;
// the compiled pack
const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
expect(fs.existsSync(path.join(compiledPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'in-pack.ql', '0.0.0', await pathSerializationBroken());
verifyQlPack('in-pack.ql', packFS.fileContents('qlpack.yml'), '0.0.0', await pathSerializationBroken());
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.false;
expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
// should have generated a correct qlpack file
const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
const qlpackContents: any = yaml.load(packFS.fileContents('qlpack.yml').toString('utf-8'));
expect(qlpackContents.name).to.equal('codeql-remote/query');
expect(qlpackContents.version).to.equal('0.0.0');
expect(qlpackContents.dependencies?.['codeql/javascript-all']).to.equal('*');
const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
const packNames = fs.readdirSync(libraryDir).sort();
const libraryDir = '.codeql/libraries/codeql';
const packNames = packFS.directoryContents(libraryDir).sort();
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
@@ -217,9 +203,14 @@ describe('Remote queries', function() {
it('should run a remote query that is nested inside a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
expect(mockSubmitRemoteQueries).to.have.been.calledOnce;
const request: RemoteQueriesSubmission = mockSubmitRemoteQueries.getCall(0).lastArg;
const packFS = await readBundledPack(request.queryPack);
// to retrieve the list of repositories
expect(showQuickPickSpy).to.have.been.calledOnce;
@@ -227,43 +218,27 @@ describe('Remote queries', function() {
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
// check a few files that we know should exist and others that we know should not
// the tarball to deliver to the server
expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
const queryPackDir = path.join(queryPackRootDir, 'query-pack');
expect(fs.existsSync(path.join(queryPackDir, 'subfolder/in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
expect(packFS.fileExists('subfolder/in-pack.ql')).to.be.true;
expect(packFS.fileExists('qlpack.yml')).to.be.true;
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
packFS.fileExists('qlpack.lock.yml') ||
packFS.fileExists('codeql-pack.lock.yml')
).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'otherfolder/lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
expect(packFS.fileExists('otherfolder/lib.qll')).to.be.true;
expect(packFS.fileExists('not-in-pack.ql')).to.be.false;
// the compiled pack
const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
expect(fs.existsSync(path.join(compiledPackDir, 'otherfolder/lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'subfolder/in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'subfolder/in-pack.ql', '0.0.0', await pathSerializationBroken());
verifyQlPack('subfolder/in-pack.ql', packFS.fileContents('qlpack.yml'), '0.0.0', await pathSerializationBroken());
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
// should have generated a correct qlpack file
const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
const qlpackContents: any = yaml.load(packFS.fileContents('qlpack.yml').toString('utf-8'));
expect(qlpackContents.name).to.equal('codeql-remote/query');
expect(qlpackContents.version).to.equal('0.0.0');
expect(qlpackContents.dependencies?.['codeql/javascript-all']).to.equal('*');
const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
const packNames = fs.readdirSync(libraryDir).sort();
const libraryDir = '.codeql/libraries/codeql';
const packNames = packFS.directoryContents(libraryDir).sort();
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
@@ -275,7 +250,7 @@ describe('Remote queries', function() {
it('should cancel a run before uploading', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const promise = runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
cancellationTokenSource.cancel();
@@ -301,7 +276,7 @@ describe('Remote queries', function() {
it('should run a variant analysis that is part of a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const variantAnalysis = querySubmissionResult!.variantAnalysis!;
expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -314,7 +289,7 @@ describe('Remote queries', function() {
it('should run a remote query that is not part of a qlpack', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const variantAnalysis = querySubmissionResult!.variantAnalysis!;
expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -327,7 +302,7 @@ describe('Remote queries', function() {
it('should run a remote query that is nested inside a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const variantAnalysis = querySubmissionResult!.variantAnalysis!;
expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -340,7 +315,7 @@ describe('Remote queries', function() {
it('should cancel a run before uploading', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const promise = runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
cancellationTokenSource.cancel();
@@ -353,8 +328,8 @@ describe('Remote queries', function() {
});
});
function verifyQlPack(qlpackPath: string, queryPath: string, packVersion: string, pathSerializationBroken: boolean) {
const qlPack = yaml.load(fs.readFileSync(qlpackPath, 'utf8')) as QlPack;
function verifyQlPack(queryPath: string, contents: Buffer, packVersion: string, pathSerializationBroken: boolean) {
const qlPack = yaml.load(contents.toString('utf-8')) as QlPack;
if (pathSerializationBroken) {
// the path serialization is broken, so we force it to be the path in the pack to be same as the query path

76
extensions/ql-vscode/src/vscode-tests/utils/bundled-pack-helpers.ts Normal file
View File

@@ -0,0 +1,76 @@
import { Readable } from 'stream';
import * as tar from 'tar-stream';
import { Headers } from 'tar-stream';
import { pipeline } from 'stream/promises';
import { createGunzip } from 'zlib';
export interface QueryPackFS {
fileExists: (name: string) => boolean;
fileContents: (name: string) => Buffer;
directoryContents: (name: string) => string[];
}
export const readBundledPack = async (base64Pack: string): Promise<QueryPackFS> => {
const buffer = Buffer.from(base64Pack, 'base64');
const stream = Readable.from(buffer);
const extract = tar.extract();
const files: Record<string, {
headers: Headers,
contents: Buffer,
}> = {};
extract.on('entry', function(headers: Headers, stream, next) {
const buffers: Buffer[] = [];
stream.on('data', chunk => buffers.push(chunk));
stream.on('end', () => {
files[headers.name] = {
headers,
contents: Buffer.concat(buffers),
};
next();
});
stream.on('error', err => {
console.error(err);
next();
});
});
await pipeline(stream, createGunzip(), extract);
const directories: Record<string, number> = {};
for (let file of Object.keys(files)) {
while (file.indexOf('/') > 0) {
const directory = file.substring(0, file.lastIndexOf('/'));
if (!(directory in directories)) {
directories[directory] = 0;
}
directories[directory]++;
file = directory;
}
}
return {
fileExists: (name: string) => {
return files[name]?.headers.type === 'file';
},
fileContents: (name: string): Buffer => {
const file = files[name];
if (file?.headers.type === 'file') {
return file.contents;
}
throw new Error(`File ${name} does not exist`);
},
directoryContents: (name: string): string[] => {
return Object.keys(directories)
.filter(dir => dir.startsWith(name) && dir !== name && dir.substring(name.length + 1).split('/').length === 1)
.map(dir => dir.substring(name.length + 1));
},
};
};