mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #1746 from github/koesie10/remote-queries-tests-tar

Browse Source 
Change remote queries to test against submitted data
This commit is contained in:
Koen Vlaswinkel
2022-11-11 11:36:06 +01:00
committed by GitHub
parent 1241ce985a a4402cb68b
commit 9894eda60c
8 changed files with 243 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
extensions/ql-vscode/package-lock.json generated
View File

@@ -96,6 +96,7 @@
"@types/sinon-chai": "~3.2.3",
"@types/stream-chain": "~2.0.1",
"@types/stream-json": "~1.7.1",
"@types/tar-stream": "^2.2.2",
"@types/through2": "^2.0.36",
"@types/tmp": "^0.1.0",
"@types/unzipper": "~0.10.1",
@@ -136,6 +137,7 @@
"proxyquire": "~2.1.3",
"sinon": "~14.0.0",
"sinon-chai": "~3.5.0",
"tar-stream": "^2.2.0",
"through2": "^4.0.2",
"ts-jest": "^29.0.1",
"ts-json-schema-generator": "^1.1.2",
@@ -13517,6 +13519,15 @@
"integrity": "sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ==",
"dev": true
},
"node_modules/@types/tar-stream": {
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/@types/tar-stream/-/tar-stream-2.2.2.tgz",
"integrity": "sha512-1AX+Yt3icFuU6kxwmPakaiGrJUwG44MpuiqPg4dSolRFk6jmvs4b3IbUol9wKDLIgU76gevn3EwE8y/DkSJCZQ==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/testing-library__jest-dom": {
"version": "5.14.5",
"resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.5.tgz",
@@ -50622,6 +50633,15 @@
"integrity": "sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ==",
"dev": true
},
"@types/tar-stream": {
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/@types/tar-stream/-/tar-stream-2.2.2.tgz",
"integrity": "sha512-1AX+Yt3icFuU6kxwmPakaiGrJUwG44MpuiqPg4dSolRFk6jmvs4b3IbUol9wKDLIgU76gevn3EwE8y/DkSJCZQ==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/testing-library__jest-dom": {
"version": "5.14.5",
"resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.5.tgz",

2
extensions/ql-vscode/package.json
View File

@@ -1380,6 +1380,7 @@
"@types/sinon-chai": "~3.2.3",
"@types/stream-chain": "~2.0.1",
"@types/stream-json": "~1.7.1",
"@types/tar-stream": "^2.2.2",
"@types/through2": "^2.0.36",
"@types/tmp": "^0.1.0",
"@types/unzipper": "~0.10.1",
@@ -1420,6 +1421,7 @@
"proxyquire": "~2.1.3",
"sinon": "~14.0.0",
"sinon-chai": "~3.5.0",
"tar-stream": "^2.2.0",
"through2": "^4.0.2",
"ts-jest": "^29.0.1",
"ts-json-schema-generator": "^1.1.2",

30
extensions/ql-vscode/src/remote-queries/gh-api/gh-api-client.ts
View File

@@ -1,5 +1,6 @@
import { Credentials } from '../../authentication';
import { OctokitResponse } from '@octokit/types/dist-types';
import { RemoteQueriesSubmission } from '../shared/remote-queries';
import { VariantAnalysisSubmission } from '../shared/variant-analysis';
import {
VariantAnalysis,
@@ -7,6 +8,7 @@ import {
VariantAnalysisSubmissionRequest
} from './variant-analysis';
import { Repository } from './repository';
import { RemoteQueriesResponse, RemoteQueriesSubmissionRequest } from './remote-queries';
export async function submitVariantAnalysis(
credentials: Credentials,
@@ -116,3 +118,31 @@ export async function createGist(
}
return response.data.html_url;
}
export async function submitRemoteQueries(
credentials: Credentials,
submissionDetails: RemoteQueriesSubmission
): Promise<RemoteQueriesResponse> {
const octokit = await credentials.getOctokit();
const { ref, language, repositories, repositoryLists, repositoryOwners, queryPack, controllerRepoId } = submissionDetails;
const data: RemoteQueriesSubmissionRequest = {
ref,
language,
repositories,
repository_lists: repositoryLists,
repository_owners: repositoryOwners,
query_pack: queryPack,
};
const response: OctokitResponse<RemoteQueriesResponse> = await octokit.request(
'POST /repositories/:controllerRepoId/code-scanning/codeql/queries',
{
controllerRepoId,
data
}
);
return response.data;
}

20
extensions/ql-vscode/src/remote-queries/gh-api/remote-queries.ts Normal file
View File

@@ -0,0 +1,20 @@
export interface RemoteQueriesSubmissionRequest {
ref: string;
language: string;
repositories?: string[];
repository_lists?: string[];
repository_owners?: string[];
query_pack: string;
}
export interface RemoteQueriesResponse {
workflow_run_id: number,
errors?: {
invalid_repositories?: string[],
repositories_without_database?: string[],
private_repositories?: string[],
cutoff_repositories?: string[],
cutoff_repositories_count?: number,
},
repositories_queried: string[],
}

42
extensions/ql-vscode/src/remote-queries/run-remote-query.ts
View File

@@ -18,13 +18,14 @@ import * as cli from '../cli';
import { logger } from '../logging';
import { getActionBranch, getRemoteControllerRepo, isVariantAnalysisLiveResultsEnabled, setRemoteControllerRepo } from '../config';
import { ProgressCallback, UserCancellationException } from '../commandRunner';
import { OctokitResponse, RequestError } from '@octokit/types/dist-types';
import { RequestError } from '@octokit/types/dist-types';
import { RemoteQuery } from './remote-query';
import { RemoteQuerySubmissionResult } from './remote-query-submission-result';
import { QueryMetadata } from '../pure/interface-types';
import { getErrorMessage, REPO_REGEX } from '../pure/helpers-pure';
import { pluralize } from '../pure/word';
import * as ghApiClient from './gh-api/gh-api-client';
import { RemoteQueriesResponse } from './gh-api/remote-queries';
import { getRepositorySelection, isValidSelection, RepositorySelection } from './repository-selection';
import { parseVariantAnalysisQueryLanguage, VariantAnalysisSubmission } from './shared/variant-analysis';
import { Repository } from './shared/repository';
@@ -39,18 +40,6 @@ export interface QlPack {
defaultSuiteFile?: string;
}
interface QueriesResponse {
workflow_run_id: number,
errors?: {
invalid_repositories?: string[],
repositories_without_database?: string[],
private_repositories?: string[],
cutoff_repositories?: string[],
cutoff_repositories_count?: number,
},
repositories_queried: string[],
}
/**
* Well-known names for the query pack used by the server.
*/
@@ -172,7 +161,7 @@ function isFileSystemRoot(dir: string): boolean {
return pathObj.root === dir && pathObj.base === '';
}
async function createRemoteQueriesTempDirectory() {
export async function createRemoteQueriesTempDirectory() {
const remoteQueryDir = await tmp.dir({ dir: tmpDir.name, unsafeCleanup: true });
const queryPackDir = path.join(remoteQueryDir.path, 'query-pack');
await fs.mkdirp(queryPackDir);
@@ -389,7 +378,7 @@ async function runRemoteQueriesApiRequest(
controllerRepo: Repository,
queryPackBase64: string,
dryRun = false
): Promise<void | QueriesResponse> {
): Promise<void | RemoteQueriesResponse> {
const data = {
ref,
language,
@@ -412,17 +401,18 @@ async function runRemoteQueriesApiRequest(
}
try {
const octokit = await credentials.getOctokit();
const response: OctokitResponse<QueriesResponse, number> = await octokit.request(
'POST /repositories/:controllerRepoId/code-scanning/codeql/queries',
{
controllerRepoId: controllerRepo.id,
data
}
);
const { popupMessage, logMessage } = parseResponse(controllerRepo, response.data);
const response = await ghApiClient.submitRemoteQueries(credentials, {
ref,
language,
repositories: repoSelection.repositories,
repositoryLists: repoSelection.repositoryLists,
repositoryOwners: repoSelection.owners,
queryPack: queryPackBase64,
controllerRepoId: controllerRepo.id,
});
const { popupMessage, logMessage } = parseResponse(controllerRepo, response);
void showAndLogInformationMessage(popupMessage, { fullMessage: logMessage });
return response.data;
return response;
} catch (error: any) {
if (error.status === 404) {
void showAndLogErrorMessage(`Controller repository was not found. Please make sure it's a valid repo name.${eol}`);
@@ -436,7 +426,7 @@ const eol = os.EOL;
const eol2 = os.EOL + os.EOL;
// exported for testing only
export function parseResponse(controllerRepo: Repository, response: QueriesResponse) {
export function parseResponse(controllerRepo: Repository, response: RemoteQueriesResponse) {
const repositoriesQueried = response.repositories_queried;
const repositoryCount = repositoriesQueried.length;

10
extensions/ql-vscode/src/remote-queries/shared/remote-queries.ts Normal file
View File

@@ -0,0 +1,10 @@
export interface RemoteQueriesSubmission {
ref: string;
language: string;
repositories?: string[];
repositoryLists?: string[];
repositoryOwners?: string[];
queryPack: string;
controllerRepoId: number;
}

163
extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/run-remote-query.test.ts
View File

@@ -2,7 +2,6 @@ import { assert, expect } from 'chai';
import * as path from 'path';
import * as sinon from 'sinon';
import { CancellationTokenSource, ExtensionContext, extensions, QuickPickItem, Uri, window } from 'vscode';
import * as fs from 'fs-extra';
import * as os from 'os';
import * as yaml from 'js-yaml';
@@ -25,6 +24,8 @@ import { createMockExtensionContext } from '../../no-workspace';
import { VariantAnalysisManager } from '../../../remote-queries/variant-analysis-manager';
import { OutputChannelLogger } from '../../../logging';
import { VariantAnalysisResultsManager } from '../../../remote-queries/variant-analysis-results-manager';
import { RemoteQueriesSubmission } from '../../../remote-queries/shared/remote-queries';
import { readBundledPack } from '../../utils/bundled-pack-helpers';
describe('Remote queries', function() {
const baseDir = path.join(__dirname, '../../../../src/vscode-tests/cli-integration');
@@ -97,12 +98,26 @@ describe('Remote queries', function() {
});
describe('when live results are not enabled', () => {
let mockSubmitRemoteQueries: sinon.SinonStub;
beforeEach(() => {
mockSubmitRemoteQueries = sandbox.stub(ghApiClient, 'submitRemoteQueries').resolves({
workflow_run_id: 20,
repositories_queried: ['octodemo/hello-world-1'],
});
});
it('should run a remote query that is part of a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
expect(mockSubmitRemoteQueries).to.have.been.calledOnce;
const request: RemoteQueriesSubmission = mockSubmitRemoteQueries.getCall(0).lastArg;
const packFS = await readBundledPack(request.queryPack);
// to retrieve the list of repositories
expect(showQuickPickSpy).to.have.been.calledOnce;
@@ -110,43 +125,25 @@ describe('Remote queries', function() {
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
// check a few files that we know should exist and others that we know should not
// the tarball to deliver to the server
expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
const queryPackDir = path.join(queryPackRootDir, 'query-pack');
expect(fs.existsSync(path.join(queryPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
expect(packFS.fileExists('in-pack.ql')).to.be.true;
expect(packFS.fileExists('lib.qll')).to.be.true;
expect(packFS.fileExists('qlpack.yml')).to.be.true;
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
packFS.fileExists('qlpack.lock.yml') ||
packFS.fileExists('codeql-pack.lock.yml')
).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
expect(packFS.fileExists('not-in-pack.ql')).to.be.false;
// the compiled pack
const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
expect(fs.existsSync(path.join(compiledPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
// should have generated a correct qlpack file
const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
const qlpackContents: any = yaml.load(packFS.fileContents('qlpack.yml').toString('utf-8'));
expect(qlpackContents.name).to.equal('codeql-remote/query');
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'in-pack.ql', '0.0.0', await pathSerializationBroken());
verifyQlPack('in-pack.ql', packFS.fileContents('qlpack.yml'), '0.0.0', await pathSerializationBroken());
const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
const packNames = fs.readdirSync(libraryDir).sort();
const libraryDir = '.codeql/libraries/codeql';
const packNames = packFS.directoryContents(libraryDir).sort();
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
@@ -158,9 +155,14 @@ describe('Remote queries', function() {
it('should run a remote query that is not part of a qlpack', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
expect(mockSubmitRemoteQueries).to.have.been.calledOnce;
const request: RemoteQueriesSubmission = mockSubmitRemoteQueries.getCall(0).lastArg;
const packFS = await readBundledPack(request.queryPack);
// to retrieve the list of repositories
// and a second time to ask for the language
@@ -169,43 +171,27 @@ describe('Remote queries', function() {
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
// check a few files that we know should exist and others that we know should not
// the tarball to deliver to the server
expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
const queryPackDir = path.join(queryPackRootDir, 'query-pack');
expect(fs.existsSync(path.join(queryPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
expect(packFS.fileExists('in-pack.ql')).to.be.true;
expect(packFS.fileExists('qlpack.yml')).to.be.true;
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
packFS.fileExists('qlpack.lock.yml') ||
packFS.fileExists('codeql-pack.lock.yml')
).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.false;
expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
expect(packFS.fileExists('lib.qll')).to.be.false;
expect(packFS.fileExists('not-in-pack.ql')).to.be.false;
// the compiled pack
const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
expect(fs.existsSync(path.join(compiledPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'in-pack.ql', '0.0.0', await pathSerializationBroken());
verifyQlPack('in-pack.ql', packFS.fileContents('qlpack.yml'), '0.0.0', await pathSerializationBroken());
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.false;
expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
// should have generated a correct qlpack file
const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
const qlpackContents: any = yaml.load(packFS.fileContents('qlpack.yml').toString('utf-8'));
expect(qlpackContents.name).to.equal('codeql-remote/query');
expect(qlpackContents.version).to.equal('0.0.0');
expect(qlpackContents.dependencies?.['codeql/javascript-all']).to.equal('*');
const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
const packNames = fs.readdirSync(libraryDir).sort();
const libraryDir = '.codeql/libraries/codeql';
const packNames = packFS.directoryContents(libraryDir).sort();
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
@@ -217,9 +203,14 @@ describe('Remote queries', function() {
it('should run a remote query that is nested inside a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
expect(mockSubmitRemoteQueries).to.have.been.calledOnce;
const request: RemoteQueriesSubmission = mockSubmitRemoteQueries.getCall(0).lastArg;
const packFS = await readBundledPack(request.queryPack);
// to retrieve the list of repositories
expect(showQuickPickSpy).to.have.been.calledOnce;
@@ -227,43 +218,27 @@ describe('Remote queries', function() {
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
// check a few files that we know should exist and others that we know should not
// the tarball to deliver to the server
expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
const queryPackDir = path.join(queryPackRootDir, 'query-pack');
expect(fs.existsSync(path.join(queryPackDir, 'subfolder/in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
expect(packFS.fileExists('subfolder/in-pack.ql')).to.be.true;
expect(packFS.fileExists('qlpack.yml')).to.be.true;
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
packFS.fileExists('qlpack.lock.yml') ||
packFS.fileExists('codeql-pack.lock.yml')
).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'otherfolder/lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
expect(packFS.fileExists('otherfolder/lib.qll')).to.be.true;
expect(packFS.fileExists('not-in-pack.ql')).to.be.false;
// the compiled pack
const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
expect(fs.existsSync(path.join(compiledPackDir, 'otherfolder/lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'subfolder/in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'subfolder/in-pack.ql', '0.0.0', await pathSerializationBroken());
verifyQlPack('subfolder/in-pack.ql', packFS.fileContents('qlpack.yml'), '0.0.0', await pathSerializationBroken());
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
// should have generated a correct qlpack file
const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
const qlpackContents: any = yaml.load(packFS.fileContents('qlpack.yml').toString('utf-8'));
expect(qlpackContents.name).to.equal('codeql-remote/query');
expect(qlpackContents.version).to.equal('0.0.0');
expect(qlpackContents.dependencies?.['codeql/javascript-all']).to.equal('*');
const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
const packNames = fs.readdirSync(libraryDir).sort();
const libraryDir = '.codeql/libraries/codeql';
const packNames = packFS.directoryContents(libraryDir).sort();
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
@@ -275,7 +250,7 @@ describe('Remote queries', function() {
it('should cancel a run before uploading', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const promise = runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
cancellationTokenSource.cancel();
@@ -301,7 +276,7 @@ describe('Remote queries', function() {
it('should run a variant analysis that is part of a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const variantAnalysis = querySubmissionResult!.variantAnalysis!;
expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -314,7 +289,7 @@ describe('Remote queries', function() {
it('should run a remote query that is not part of a qlpack', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const variantAnalysis = querySubmissionResult!.variantAnalysis!;
expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -327,7 +302,7 @@ describe('Remote queries', function() {
it('should run a remote query that is nested inside a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const variantAnalysis = querySubmissionResult!.variantAnalysis!;
expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -340,7 +315,7 @@ describe('Remote queries', function() {
it('should cancel a run before uploading', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
const promise = runRemoteQuery(cli, credentials, fileUri, false, progress, cancellationTokenSource.token, variantAnalysisManager);
cancellationTokenSource.cancel();
@@ -353,8 +328,8 @@ describe('Remote queries', function() {
});
});
function verifyQlPack(qlpackPath: string, queryPath: string, packVersion: string, pathSerializationBroken: boolean) {
const qlPack = yaml.load(fs.readFileSync(qlpackPath, 'utf8')) as QlPack;
function verifyQlPack(queryPath: string, contents: Buffer, packVersion: string, pathSerializationBroken: boolean) {
const qlPack = yaml.load(contents.toString('utf-8')) as QlPack;
if (pathSerializationBroken) {
// the path serialization is broken, so we force it to be the path in the pack to be same as the query path

76
extensions/ql-vscode/src/vscode-tests/utils/bundled-pack-helpers.ts Normal file
View File

@@ -0,0 +1,76 @@
import { Readable } from 'stream';
import * as tar from 'tar-stream';
import { Headers } from 'tar-stream';
import { pipeline } from 'stream/promises';
import { createGunzip } from 'zlib';
export interface QueryPackFS {
fileExists: (name: string) => boolean;
fileContents: (name: string) => Buffer;
directoryContents: (name: string) => string[];
}
export async function readBundledPack(base64Pack: string): Promise<QueryPackFS> {
const buffer = Buffer.from(base64Pack, 'base64');
const stream = Readable.from(buffer);
const extract = tar.extract();
const files: Record<string, {
headers: Headers,
contents: Buffer,
}> = {};
extract.on('entry', function(headers: Headers, stream, next) {
const buffers: Buffer[] = [];
stream.on('data', chunk => buffers.push(chunk));
stream.on('end', () => {
files[headers.name] = {
headers,
contents: Buffer.concat(buffers),
};
next();
});
stream.on('error', err => {
console.error(err);
next();
});
});
await pipeline(stream, createGunzip(), extract);
const directories: Record<string, number> = {};
for (let file of Object.keys(files)) {
while (file.indexOf('/') > 0) {
const directory = file.substring(0, file.lastIndexOf('/'));
if (!(directory in directories)) {
directories[directory] = 0;
}
directories[directory]++;
file = directory;
}
}
return {
fileExists: (name: string) => {
return files[name]?.headers.type === 'file';
},
fileContents: (name: string): Buffer => {
const file = files[name];
if (file?.headers.type === 'file') {
return file.contents;
}
throw new Error(`File ${name} does not exist`);
},
directoryContents: (name: string): string[] => {
return Object.keys(directories)
.filter(dir => dir.startsWith(name) && dir !== name && dir.substring(name.length + 1).split('/').length === 1)
.map(dir => dir.substring(name.length + 1));
},
};
}