Merge remote-tracking branch 'origin/main' into koesie10/request-repo-results-message

This commit is contained in:
Koen Vlaswinkel
2022-10-11 11:08:49 +02:00
11 changed files with 120 additions and 74 deletions

View File

@@ -467,16 +467,16 @@ async function activateWithInstalledDistribution(
const localQueryResultsView = new ResultsView(ctx, dbm, cliServer, queryServerLogger, labelProvider);
ctx.subscriptions.push(localQueryResultsView);
void logger.log('Initializing remote queries manager.');
const rqm = new RemoteQueriesManager(ctx, cliServer, queryStorageDir, logger);
ctx.subscriptions.push(rqm);
void logger.log('Initializing variant analysis manager.');
const variantAnalysisStorageDir = path.join(ctx.globalStorageUri.fsPath, 'variant-analyses');
await fs.ensureDir(variantAnalysisStorageDir);
const variantAnalysisManager = new VariantAnalysisManager(ctx, cliServer, variantAnalysisStorageDir, logger);
ctx.subscriptions.push(variantAnalysisManager);
void logger.log('Initializing remote queries manager.');
const rqm = new RemoteQueriesManager(ctx, cliServer, queryStorageDir, logger, variantAnalysisManager);
ctx.subscriptions.push(rqm);
void logger.log('Initializing query history.');
const qhm = new QueryHistoryManager(
qs,

View File

@@ -77,12 +77,9 @@ export async function getVariantAnalysisRepo(
export async function getVariantAnalysisRepoResult(
credentials: Credentials,
downloadUrl: string,
): Promise<unknown> {
): Promise<ArrayBuffer> {
const octokit = await credentials.getOctokit();
const response: OctokitResponse<VariantAnalysisRepoTask> = await octokit.request(
`GET ${downloadUrl}`
);
const response = await octokit.request(`GET ${downloadUrl}`);
return response.data;
}

View File

@@ -22,6 +22,7 @@ import { assertNever } from '../pure/helpers-pure';
import { QueryStatus } from '../query-status';
import { DisposableObject } from '../pure/disposable-object';
import { AnalysisResults } from './shared/analysis-result';
import { VariantAnalysisManager } from './variant-analysis-manager';
const autoDownloadMaxSize = 300 * 1024;
const autoDownloadMaxCount = 100;
@@ -56,6 +57,7 @@ export class RemoteQueriesManager extends DisposableObject {
private readonly remoteQueriesMonitor: RemoteQueriesMonitor;
private readonly analysesResultsManager: AnalysesResultsManager;
private readonly variantAnalysisManager: VariantAnalysisManager;
private readonly view: RemoteQueriesView;
constructor(
@@ -63,11 +65,13 @@ export class RemoteQueriesManager extends DisposableObject {
private readonly cliServer: CodeQLCliServer,
private readonly storagePath: string,
logger: Logger,
variantAnalysisManager: VariantAnalysisManager,
) {
super();
this.analysesResultsManager = new AnalysesResultsManager(ctx, cliServer, storagePath, logger);
this.view = new RemoteQueriesView(ctx, logger, this.analysesResultsManager);
this.remoteQueriesMonitor = new RemoteQueriesMonitor(ctx, logger);
this.variantAnalysisManager = variantAnalysisManager;
this.remoteQueryAddedEventEmitter = this.push(new EventEmitter<NewQueryEvent>());
this.remoteQueryRemovedEventEmitter = this.push(new EventEmitter<RemovedQueryEvent>());
@@ -123,7 +127,8 @@ export class RemoteQueriesManager extends DisposableObject {
credentials, uri || window.activeTextEditor?.document.uri,
false,
progress,
token);
token,
this.variantAnalysisManager);
if (querySubmission?.query) {
const query = querySubmission.query;

View File

@@ -29,6 +29,7 @@ import { getRepositorySelection, isValidSelection, RepositorySelection } from '.
import { parseVariantAnalysisQueryLanguage, VariantAnalysisSubmission } from './shared/variant-analysis';
import { Repository } from './shared/repository';
import { processVariantAnalysis } from './variant-analysis-processor';
import { VariantAnalysisManager } from './variant-analysis-manager';
export interface QlPack {
name: string;
@@ -182,7 +183,8 @@ export async function runRemoteQuery(
uri: Uri | undefined,
dryRun: boolean,
progress: ProgressCallback,
token: CancellationToken
token: CancellationToken,
variantAnalysisManager: VariantAnalysisManager
): Promise<void | RemoteQuerySubmissionResult> {
if (!(await cliServer.cliConstraints.supportsRemoteQueries())) {
throw new Error(`Variant analysis is not supported by this version of CodeQL. Please upgrade to v${cli.CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES
@@ -273,6 +275,8 @@ export async function runRemoteQuery(
const processedVariantAnalysis = processVariantAnalysis(variantAnalysisSubmission, variantAnalysisResponse);
variantAnalysisManager.onVariantAnalysisSubmitted(processedVariantAnalysis);
void logger.log(`Variant analysis:\n${JSON.stringify(processedVariantAnalysis, null, 2)}`);
void showAndLogInformationMessage(`Variant analysis ${processedVariantAnalysis.query.name} submitted for processing`);

View File

@@ -1,5 +1,5 @@
import * as ghApiClient from './gh-api/gh-api-client';
import { CancellationToken, ExtensionContext } from 'vscode';
import { CancellationToken, EventEmitter, ExtensionContext } from 'vscode';
import { DisposableObject } from '../pure/disposable-object';
import { Logger } from '../logging';
import { Credentials } from '../authentication';
@@ -22,6 +22,9 @@ import { VariantAnalysisResultsManager } from './variant-analysis-results-manage
import { CodeQLCliServer } from '../cli';
export class VariantAnalysisManager extends DisposableObject implements VariantAnalysisViewManager<VariantAnalysisView> {
private readonly _onVariantAnalysisAdded = this.push(new EventEmitter<VariantAnalysis | undefined>());
readonly onVariantAnalysisAdded = this._onVariantAnalysisAdded.event;
private readonly variantAnalysisMonitor: VariantAnalysisMonitor;
private readonly variantAnalysisResultsManager: VariantAnalysisResultsManager;
private readonly variantAnalyses = new Map<number, VariantAnalysis>();
@@ -87,6 +90,10 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
await this.getView(variantAnalysis.id)?.updateView(variantAnalysis);
}
public onVariantAnalysisSubmitted(variantAnalysis: VariantAnalysis): void {
this._onVariantAnalysisAdded.fire(variantAnalysis);
}
private async onRepoResultLoaded(repositoryResult: VariantAnalysisScannedRepositoryResult): Promise<void> {
await this.getView(repositoryResult.variantAnalysisId)?.sendRepositoryResults([repositoryResult]);
}

View File

@@ -14,6 +14,7 @@ import { DisposableObject, DisposeHandler } from '../pure/disposable-object';
import { VariantAnalysisRepoTask } from './gh-api/variant-analysis';
import * as ghApiClient from './gh-api/gh-api-client';
import { EventEmitter } from 'vscode';
import { unzipFile } from '../pure/zip';
type CacheKey = `${number}/${string}`;
@@ -60,9 +61,17 @@ export class VariantAnalysisResultsManager extends DisposableObject {
repoTask.artifact_url
);
fs.mkdirSync(resultDirectory, { recursive: true });
if (!(await fs.pathExists(resultDirectory))) {
await fs.mkdir(resultDirectory, { recursive: true });
}
await fs.outputJson(path.join(resultDirectory, VariantAnalysisResultsManager.REPO_TASK_FILENAME), repoTask);
await fs.writeFile(path.join(resultDirectory, 'results.zip'), JSON.stringify(result, null, 2), 'utf8');
const zipFilePath = path.join(resultDirectory, 'results.zip');
const unzippedFilesDirectory = path.join(resultDirectory, 'results');
fs.writeFileSync(zipFilePath, Buffer.from(result));
await unzipFile(zipFilePath, unzippedFilesDirectory);
this._onResultDownloaded.fire({
variantAnalysisId,
@@ -161,7 +170,7 @@ export class VariantAnalysisResultsManager extends DisposableObject {
);
}
private getRepoStorageDirectory(variantAnalysisId: number, fullName: string): string {
public getRepoStorageDirectory(variantAnalysisId: number, fullName: string): string {
return path.join(
this.getStorageDirectory(variantAnalysisId),
fullName

View File

@@ -1,7 +1,7 @@
import { assert, expect } from 'chai';
import * as path from 'path';
import * as sinon from 'sinon';
import { CancellationTokenSource, extensions, QuickPickItem, Uri, window } from 'vscode';
import { CancellationTokenSource, ExtensionContext, extensions, QuickPickItem, Uri, window } from 'vscode';
import * as fs from 'fs-extra';
import * as os from 'os';
import * as yaml from 'js-yaml';
@@ -21,6 +21,9 @@ import {
import { Repository } from '../../../remote-queries/gh-api/repository';
import { VariantAnalysisStatus } from '../../../remote-queries/shared/variant-analysis';
import { createMockApiResponse } from '../../factories/remote-queries/gh-api/variant-analysis-api-response';
import { createMockExtensionContext } from '../../no-workspace';
import { VariantAnalysisManager } from '../../../remote-queries/variant-analysis-manager';
import { OutputChannelLogger } from '../../../logging';
describe('Remote queries', function() {
const baseDir = path.join(__dirname, '../../../../src/vscode-tests/cli-integration');
@@ -37,6 +40,9 @@ describe('Remote queries', function() {
let showQuickPickSpy: sinon.SinonStub;
let getRepositoryFromNwoStub: sinon.SinonStub;
let liveResultsStub: sinon.SinonStub;
let ctx: ExtensionContext;
let logger: any;
let variantAnalysisManager: VariantAnalysisManager;
// use `function` so we have access to `this`
beforeEach(async function() {
@@ -49,6 +55,10 @@ describe('Remote queries', function() {
throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
}
ctx = createMockExtensionContext();
logger = new OutputChannelLogger('test-logger');
variantAnalysisManager = new VariantAnalysisManager(ctx, cli, 'fake-storage-dir', logger);
if (!(await cli.cliConstraints.supportsRemoteQueries())) {
console.log(`Remote queries are not supported on CodeQL CLI v${CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES
}. Skipping this test.`);
@@ -56,14 +66,7 @@ describe('Remote queries', function() {
}
credentials = {} as unknown as Credentials;
cancellationTokenSource = {
token: {
isCancellationRequested: false,
onCancellationRequested: sandbox.stub()
},
cancel: sandbox.stub(),
dispose: sandbox.stub()
};
cancellationTokenSource = new CancellationTokenSource();
progress = sandbox.spy();
// Should not have asked for a language
@@ -94,7 +97,7 @@ describe('Remote queries', function() {
it('should run a remote query that is part of a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
printDirectoryContents(queryPackRootDir);
@@ -155,7 +158,7 @@ describe('Remote queries', function() {
it('should run a remote query that is not part of a qlpack', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
@@ -218,7 +221,7 @@ describe('Remote queries', function() {
it('should run a remote query that is nested inside a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
@@ -280,9 +283,9 @@ describe('Remote queries', function() {
it('should cancel a run before uploading', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
cancellationTokenSource.token.isCancellationRequested = true;
cancellationTokenSource.cancel();
try {
await promise;
@@ -306,7 +309,7 @@ describe('Remote queries', function() {
it('should run a variant analysis that is part of a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const variantAnalysis = querySubmissionResult!.variantAnalysis!;
expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -319,7 +322,7 @@ describe('Remote queries', function() {
it('should run a remote query that is not part of a qlpack', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const variantAnalysis = querySubmissionResult!.variantAnalysis!;
expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -332,7 +335,7 @@ describe('Remote queries', function() {
it('should run a remote query that is nested inside a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
expect(querySubmissionResult).to.be.ok;
const variantAnalysis = querySubmissionResult!.variantAnalysis!;
expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -345,9 +348,9 @@ describe('Remote queries', function() {
it('should cancel a run before uploading', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token, variantAnalysisManager);
cancellationTokenSource.token.isCancellationRequested = true;
cancellationTokenSource.cancel();
try {
await promise;

View File

@@ -7,10 +7,12 @@ import * as config from '../../../config';
import * as ghApiClient from '../../../remote-queries/gh-api/gh-api-client';
import { Credentials } from '../../../authentication';
import * as fs from 'fs-extra';
import * as path from 'path';
import { VariantAnalysisManager } from '../../../remote-queries/variant-analysis-manager';
import {
VariantAnalysis as VariantAnalysisApiResponse,
VariantAnalysisRepoTask,
VariantAnalysisScannedRepository as ApiVariantAnalysisScannedRepository
} from '../../../remote-queries/gh-api/variant-analysis';
import { createMockApiResponse } from '../../factories/remote-queries/gh-api/variant-analysis-api-response';
@@ -36,14 +38,7 @@ describe('Variant Analysis Manager', async function() {
sandbox.stub(fs, 'mkdirSync');
sandbox.stub(fs, 'writeFile');
cancellationTokenSource = {
token: {
isCancellationRequested: false,
onCancellationRequested: sandbox.stub()
},
cancel: sandbox.stub(),
dispose: sandbox.stub()
};
cancellationTokenSource = new CancellationTokenSource();
scannedRepos = createMockScannedRepos();
variantAnalysis = createMockApiResponse('in_progress', scannedRepos);
@@ -79,6 +74,7 @@ describe('Variant Analysis Manager', async function() {
describe('when credentials are valid', async () => {
let getOctokitStub: sinon.SinonStub;
let arrayBuffer: ArrayBuffer;
beforeEach(async () => {
const mockCredentials = {
@@ -87,16 +83,18 @@ describe('Variant Analysis Manager', async function() {
})
} as unknown as Credentials;
sandbox.stub(Credentials, 'initialize').resolves(mockCredentials);
const sourceFilePath = path.join(__dirname, '../../../../src/vscode-tests/cli-integration/data/variant-analysis-results.zip');
arrayBuffer = fs.readFileSync(sourceFilePath).buffer;
});
describe('when the artifact_url is missing', async () => {
beforeEach(async () => {
const dummyRepoTask = createMockVariantAnalysisRepoTask();
delete dummyRepoTask.artifact_url;
getVariantAnalysisRepoStub = sandbox.stub(ghApiClient, 'getVariantAnalysisRepo').resolves(dummyRepoTask);
const dummyResult = 'this-is-a-repo-result';
getVariantAnalysisRepoResultStub = sandbox.stub(ghApiClient, 'getVariantAnalysisRepoResult').resolves(dummyResult);
getVariantAnalysisRepoStub = sandbox.stub(ghApiClient, 'getVariantAnalysisRepo').resolves(dummyRepoTask);
getVariantAnalysisRepoResultStub = sandbox.stub(ghApiClient, 'getVariantAnalysisRepoResult').resolves(arrayBuffer);
});
it('should not try to download the result', async () => {
@@ -111,16 +109,17 @@ describe('Variant Analysis Manager', async function() {
});
describe('when the artifact_url is present', async () => {
beforeEach(async () => {
const dummyRepoTask = createMockVariantAnalysisRepoTask();
getVariantAnalysisRepoStub = sandbox.stub(ghApiClient, 'getVariantAnalysisRepo').resolves(dummyRepoTask);
let dummyRepoTask: VariantAnalysisRepoTask;
const dummyResult = 'this-is-a-repo-result';
getVariantAnalysisRepoResultStub = sandbox.stub(ghApiClient, 'getVariantAnalysisRepoResult').resolves(dummyResult);
beforeEach(async () => {
dummyRepoTask = createMockVariantAnalysisRepoTask();
getVariantAnalysisRepoStub = sandbox.stub(ghApiClient, 'getVariantAnalysisRepo').resolves(dummyRepoTask);
getVariantAnalysisRepoResultStub = sandbox.stub(ghApiClient, 'getVariantAnalysisRepoResult').resolves(arrayBuffer);
});
it('should return early if variant analysis is cancelled', async () => {
cancellationTokenSource.token.isCancellationRequested = true;
cancellationTokenSource.cancel();
await variantAnalysisManager.autoDownloadVariantAnalysisResult(
scannedRepos[0],
@@ -150,16 +149,6 @@ describe('Variant Analysis Manager', async function() {
expect(getVariantAnalysisRepoResultStub.calledOnce).to.be.true;
});
it('should save the result to disk', async () => {
await variantAnalysisManager.autoDownloadVariantAnalysisResult(
scannedRepos[0],
variantAnalysis,
cancellationTokenSource.token
);
expect(getVariantAnalysisRepoResultStub.calledOnce).to.be.true;
});
});
});
});

View File

@@ -31,14 +31,7 @@ describe('Variant Analysis Monitor', async function() {
sandbox.stub(logger, 'log');
sandbox.stub(config, 'isVariantAnalysisLiveResultsEnabled').returns(false);
cancellationTokenSource = {
token: {
isCancellationRequested: false,
onCancellationRequested: sandbox.stub()
},
cancel: sandbox.stub(),
dispose: sandbox.stub()
};
cancellationTokenSource = new CancellationTokenSource();
variantAnalysis = createMockVariantAnalysis();
@@ -79,7 +72,7 @@ describe('Variant Analysis Monitor', async function() {
});
it('should return early if variant analysis is cancelled', async () => {
cancellationTokenSource.token.isCancellationRequested = true;
cancellationTokenSource.cancel();
const result = await variantAnalysisMonitor.monitorVariantAnalysis(variantAnalysis, cancellationTokenSource.token);

View File

@@ -5,6 +5,7 @@ import { CodeQLExtensionInterface } from '../../../extension';
import { logger } from '../../../logging';
import { Credentials } from '../../../authentication';
import * as fs from 'fs-extra';
import * as path from 'path';
import { VariantAnalysisResultsManager } from '../../../remote-queries/variant-analysis-results-manager';
import { createMockVariantAnalysisRepoTask } from '../../factories/remote-queries/gh-api/variant-analysis-repo-task';
@@ -12,6 +13,7 @@ import { CodeQLCliServer } from '../../../cli';
import { storagePath } from '../global.helper';
import { faker } from '@faker-js/faker';
import * as ghApiClient from '../../../remote-queries/gh-api/gh-api-client';
import { VariantAnalysisRepoTask } from '../../../remote-queries/gh-api/variant-analysis';
describe(VariantAnalysisResultsManager.name, () => {
let sandbox: sinon.SinonSandbox;
@@ -69,12 +71,29 @@ describe(VariantAnalysisResultsManager.name, () => {
});
describe('when the artifact_url is present', async () => {
it('should save the result to disk', async () => {
const dummyRepoTask = createMockVariantAnalysisRepoTask();
let dummyRepoTask: VariantAnalysisRepoTask;
let storageDirectory: string;
let arrayBuffer: ArrayBuffer;
const dummyResult = 'this-is-a-repo-result';
getVariantAnalysisRepoResultStub = sandbox.stub(ghApiClient, 'getVariantAnalysisRepoResult').withArgs(mockCredentials, dummyRepoTask.artifact_url as string).resolves(dummyResult);
beforeEach(async () => {
dummyRepoTask = createMockVariantAnalysisRepoTask();
storageDirectory = variantAnalysisResultsManager.getRepoStorageDirectory(variantAnalysisId, dummyRepoTask.repository.full_name);
const sourceFilePath = path.join(__dirname, '../../../../src/vscode-tests/cli-integration/data/variant-analysis-results.zip');
arrayBuffer = fs.readFileSync(sourceFilePath).buffer;
getVariantAnalysisRepoResultStub = sandbox
.stub(ghApiClient, 'getVariantAnalysisRepoResult')
.withArgs(mockCredentials, dummyRepoTask.artifact_url as string)
.resolves(arrayBuffer);
});
afterEach(async () => {
fs.removeSync(`${storageDirectory}/results.zip`);
fs.removeSync(`${storageDirectory}/results`);
});
it('should call the API to download the results', async () => {
await variantAnalysisResultsManager.download(
mockCredentials,
variantAnalysisId,
@@ -83,6 +102,26 @@ describe(VariantAnalysisResultsManager.name, () => {
expect(getVariantAnalysisRepoResultStub.calledOnce).to.be.true;
});
it('should save the results zip file to disk', async () => {
await variantAnalysisResultsManager.download(
mockCredentials,
variantAnalysisId,
dummyRepoTask
);
expect(fs.existsSync(`${storageDirectory}/results.zip`)).to.be.true;
});
it('should unzip the results in a `results/` folder', async () => {
await variantAnalysisResultsManager.download(
mockCredentials,
variantAnalysisId,
dummyRepoTask
);
expect(fs.existsSync(`${storageDirectory}/results/results.sarif`)).to.be.true;
});
});
});
});