mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into dbartol/goto-ql

Browse Source
This commit is contained in:
Dave Bartolomeo
2022-07-19 09:39:51 -04:00
parent 49a1576d14 2cf5b39cfe
commit 84048ccac1
35 changed files with 1095 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
extensions/ql-vscode/package-lock.json generated
View File

@@ -9,9 +9,11 @@
"version": "1.6.9",
"license": "MIT",
"dependencies": {
"@octokit/plugin-retry": "^3.0.9",
"@octokit/rest": "^18.5.6",
"@primer/octicons-react": "^16.3.0",
"@primer/react": "^35.0.0",
"@vscode/codicons": "^0.0.31",
"@vscode/webview-ui-toolkit": "^1.0.0",
"child-process-promise": "^2.2.1",
"classnames": "~2.2.6",
@@ -697,6 +699,15 @@
"@octokit/core": ">=3"
}
},
"node_modules/@octokit/plugin-retry": {
"version": "3.0.9",
"resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-3.0.9.tgz",
"integrity": "sha512-r+fArdP5+TG6l1Rv/C9hVoty6tldw6cE2pRHNGmFPdyfrc696R6JjrQ3d7HdVqGwuzfyrcaLAKD7K8TX8aehUQ==",
"dependencies": {
"@octokit/types": "^6.0.3",
"bottleneck": "^2.15.3"
}
},
"node_modules/@octokit/request": {
"version": "5.6.0",
"resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.0.tgz",
@@ -2151,6 +2162,11 @@
"integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==",
"dev": true
},
"node_modules/@vscode/codicons": {
"version": "0.0.31",
"resolved": "https://registry.npmjs.org/@vscode/codicons/-/codicons-0.0.31.tgz",
"integrity": "sha512-fldpXy7pHsQAMlU1pnGI23ypQ6xLk5u6SiABMFoAmlj4f2MR0iwg7C19IB1xvAEGG+dkxOfRSrbKF8ry7QqGQA=="
},
"node_modules/@vscode/webview-ui-toolkit": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@vscode/webview-ui-toolkit/-/webview-ui-toolkit-1.0.0.tgz",
@@ -3174,6 +3190,11 @@
"integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=",
"dev": true
},
"node_modules/bottleneck": {
"version": "2.19.5",
"resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz",
"integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="
},
"node_modules/brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -14724,6 +14745,15 @@
"deprecation": "^2.3.1"
}
},
"@octokit/plugin-retry": {
"version": "3.0.9",
"resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-3.0.9.tgz",
"integrity": "sha512-r+fArdP5+TG6l1Rv/C9hVoty6tldw6cE2pRHNGmFPdyfrc696R6JjrQ3d7HdVqGwuzfyrcaLAKD7K8TX8aehUQ==",
"requires": {
"@octokit/types": "^6.0.3",
"bottleneck": "^2.15.3"
}
},
"@octokit/request": {
"version": "5.6.0",
"resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.0.tgz",
@@ -16016,6 +16046,11 @@
"integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==",
"dev": true
},
"@vscode/codicons": {
"version": "0.0.31",
"resolved": "https://registry.npmjs.org/@vscode/codicons/-/codicons-0.0.31.tgz",
"integrity": "sha512-fldpXy7pHsQAMlU1pnGI23ypQ6xLk5u6SiABMFoAmlj4f2MR0iwg7C19IB1xvAEGG+dkxOfRSrbKF8ry7QqGQA=="
},
"@vscode/webview-ui-toolkit": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@vscode/webview-ui-toolkit/-/webview-ui-toolkit-1.0.0.tgz",
@@ -16860,6 +16895,11 @@
"integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=",
"dev": true
},
"bottleneck": {
"version": "2.19.5",
"resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz",
"integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="
},
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",

6
extensions/ql-vscode/package.json
View File

@@ -231,7 +231,7 @@
},
"codeQL.queryHistory.format": {
"type": "string",
"default": "%q on %d - %s, %r [%t]",
"default": "%q on %d - %s %r [%t]",
"markdownDescription": "Default string for how to label query history items.\n* %t is the time of the query\n* %q is the human-readable query name\n* %f is the query file name\n* %d is the database name\n* %r is the number of results\n* %s is a status string"
},
"codeQL.queryHistory.ttl": {
@@ -1160,8 +1160,10 @@
},
"dependencies": {
"@octokit/rest": "^18.5.6",
"@octokit/plugin-retry": "^3.0.9",
"@primer/octicons-react": "^16.3.0",
"@primer/react": "^35.0.0",
"@vscode/codicons": "^0.0.31",
"@vscode/webview-ui-toolkit": "^1.0.0",
"child-process-promise": "^2.2.1",
"classnames": "~2.2.6",
@@ -1177,8 +1179,8 @@
"react": "^17.0.2",
"react-dom": "^17.0.2",
"semver": "~7.3.2",
"source-map-support": "^0.5.21",
"source-map": "^0.7.4",
"source-map-support": "^0.5.21",
"stream": "^0.0.2",
"stream-chain": "~2.2.4",
"stream-json": "~1.7.3",

6
extensions/ql-vscode/src/authentication.ts
View File

@@ -1,5 +1,6 @@
import * as vscode from 'vscode';
import * as Octokit from '@octokit/rest';
import { retry } from '@octokit/plugin-retry';
const GITHUB_AUTH_PROVIDER_ID = 'github';
@@ -51,14 +52,15 @@ export class Credentials {
private async createOctokit(createIfNone: boolean, overrideToken?: string): Promise<Octokit.Octokit | undefined> {
if (overrideToken) {
return new Octokit.Octokit({ auth: overrideToken });
return new Octokit.Octokit({ auth: overrideToken, retry });
}
const session = await vscode.authentication.getSession(GITHUB_AUTH_PROVIDER_ID, SCOPES, { createIfNone });
if (session) {
return new Octokit.Octokit({
auth: session.accessToken
auth: session.accessToken,
retry
});
} else {
return undefined;

17
extensions/ql-vscode/src/cli.ts
View File

@@ -690,6 +690,23 @@ export class CodeQLCliServer implements Disposable {
return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating log summary');
}
/**
* Generate a JSON summary of an evaluation log.
* @param inputPath The path of an evaluation event log.
* @param outputPath The path to write a JSON summary of it to.
*/
async generateJsonLogSummary(
inputPath: string,
outputPath: string,
): Promise<string> {
const subcommandArgs = [
'--format=predicates',
inputPath,
outputPath
];
return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating JSON log summary');
}
/**
* Gets the results from a bqrs.
* @param bqrsPath The path to the bqrs.

2
extensions/ql-vscode/src/config.ts
View File

@@ -59,7 +59,7 @@ const PERSONAL_ACCESS_TOKEN_SETTING = new Setting('personalAccessToken', DISTRIB
// Query History configuration
const QUERY_HISTORY_SETTING = new Setting('queryHistory', ROOT_SETTING);
const QUERY_HISTORY_FORMAT_SETTING = new Setting('format', QUERY_HISTORY_SETTING);
const QUERY_HISTORY_TTL = new Setting('format', QUERY_HISTORY_SETTING);
const QUERY_HISTORY_TTL = new Setting('ttl', QUERY_HISTORY_SETTING);
/** When these settings change, the distribution should be updated. */
const DISTRIBUTION_CHANGE_SETTINGS = [CUSTOM_CODEQL_PATH_SETTING, INCLUDE_PRERELEASE_SETTING, PERSONAL_ACCESS_TOKEN_SETTING];

8
extensions/ql-vscode/src/helpers.ts
View File

@@ -581,3 +581,11 @@ export async function* walkDirectory(dir: string): AsyncIterableIterator<string>
}
}
}
/**
* Pluralizes a word.
* Example: Returns "N repository" if N is one, "N repositories" otherwise.
*/
export function pluralize(numItems: number | undefined, singular: string, plural: string): string {
return numItems ? `${numItems} ${numItems === 1 ? singular : plural}` : '';
}

24
extensions/ql-vscode/src/history-item-label-provider.ts
View File

@@ -3,6 +3,7 @@ import * as path from 'path';
import { QueryHistoryConfig } from './config';
import { LocalQueryInfo, QueryHistoryInfo } from './query-results';
import { RemoteQueryHistoryItem } from './remote-queries/remote-query-history-item';
import { pluralize } from './helpers';
interface InterpolateReplacements {
t: string; // Start time
@@ -57,23 +58,30 @@ export class HistoryItemLabelProvider {
t: item.startTime,
q: item.getQueryName(),
d: item.initialInfo.databaseInfo.name,
r: `${resultCount} results`,
r: `(${resultCount} results)`,
s: statusString,
f: item.getQueryFileName(),
'%': '%',
};
}
// Return the number of repositories queried if available. Otherwise, use the controller repository name.
private buildRepoLabel(item: RemoteQueryHistoryItem): string {
const repositoryCount = item.remoteQuery.repositoryCount;
if (repositoryCount) {
return pluralize(repositoryCount, 'repository', 'repositories');
}
return `${item.remoteQuery.controllerRepository.owner}/${item.remoteQuery.controllerRepository.name}`;
}
private getRemoteInterpolateReplacements(item: RemoteQueryHistoryItem): InterpolateReplacements {
return {
t: new Date(item.remoteQuery.executionStartTime).toLocaleString(env.language),
q: item.remoteQuery.queryName,
// There is no database name for remote queries. Instead use the controller repository name.
d: `${item.remoteQuery.controllerRepository.owner}/${item.remoteQuery.controllerRepository.name}`,
// There is no synchronous way to get the results count.
r: '',
q: `${item.remoteQuery.queryName} (${item.remoteQuery.language})`,
d: this.buildRepoLabel(item),
r: `(${pluralize(item.resultCount, 'result', 'results')})`,
s: item.status,
f: path.basename(item.remoteQuery.queryFilePath),
'%': '%'

4
extensions/ql-vscode/src/interface-utils.ts
View File

@@ -137,6 +137,8 @@ export function getHtmlForWebview(
? `${webview.cspSource} vscode-file: 'unsafe-inline'`
: `'nonce-${nonce}'`;
const fontSrc = webview.cspSource;
/*
* Content security policy:
* default-src: allow nothing by default.
@@ -149,7 +151,7 @@ export function getHtmlForWebview(
<html>
<head>
<meta http-equiv="Content-Security-Policy"
content="default-src 'none'; script-src 'nonce-${nonce}'; style-src ${styleSrc}; connect-src ${webview.cspSource};">
content="default-src 'none'; script-src 'nonce-${nonce}'; font-src ${fontSrc}; style-src ${styleSrc}; connect-src ${webview.cspSource};">
${stylesheetsHtmlLines.join(` ${os.EOL}`)}
</head>
<body>

44
extensions/ql-vscode/src/pure/log-summary-parser.ts Normal file
View File

@@ -0,0 +1,44 @@
import * as os from 'os';
// TODO(angelapwen): Only load in necessary information and
// location in bytes for this log to save memory.
export interface EvaluatorLogData {
queryCausingWork: string;
predicateName: string;
millis: number;
resultSize: number;
ra: Pipelines;
}
interface Pipelines {
// Key: pipeline identifier; Value: array of pipeline steps
pipelineNamesToSteps: Map<string, string[]>;
}
/**
* A pure method that parses a string of evaluator log summaries into
* an array of EvaluatorLogData objects.
*
*/
export function parseVisualizerData(logSummary: string): EvaluatorLogData[] {
// Remove newline delimiters because summary is in .jsonl format.
const jsonSummaryObjects: string[] = logSummary.split(os.EOL + os.EOL);
const visualizerData: EvaluatorLogData[] = [];
for (const obj of jsonSummaryObjects) {
const jsonObj = JSON.parse(obj);
// Only convert log items that have an RA and millis field
if (jsonObj.ra !== undefined && jsonObj.millis !== undefined) {
const newLogData: EvaluatorLogData = {
queryCausingWork: jsonObj.queryCausingWork,
predicateName: jsonObj.predicateName,
millis: jsonObj.millis,
resultSize: jsonObj.resultSize,
ra: jsonObj.ra
};
visualizerData.push(newLogData);
}
}
return visualizerData;
}

6
extensions/ql-vscode/src/query-history-scrubber.ts
View File

@@ -3,6 +3,7 @@ import * as os from 'os';
import * as path from 'path';
import { Disposable, ExtensionContext } from 'vscode';
import { logger } from './logging';
import { QueryHistoryManager } from './query-history';
const LAST_SCRUB_TIME_KEY = 'lastScrubTime';
@@ -30,12 +31,13 @@ export function registerQueryHistoryScubber(
throttleTime: number,
maxQueryTime: number,
queryDirectory: string,
qhm: QueryHistoryManager,
ctx: ExtensionContext,
// optional counter to keep track of how many times the scrubber has run
counter?: Counter
): Disposable {
const deregister = setInterval(scrubQueries, wakeInterval, throttleTime, maxQueryTime, queryDirectory, ctx, counter);
const deregister = setInterval(scrubQueries, wakeInterval, throttleTime, maxQueryTime, queryDirectory, qhm, ctx, counter);
return {
dispose: () => {
@@ -48,6 +50,7 @@ async function scrubQueries(
throttleTime: number,
maxQueryTime: number,
queryDirectory: string,
qhm: QueryHistoryManager,
ctx: ExtensionContext,
counter?: Counter
) {
@@ -89,6 +92,7 @@ async function scrubQueries(
} finally {
void logger.log(`Scrubbed ${scrubCount} old queries.`);
}
await qhm.removeDeletedQueries();
}
}

25
extensions/ql-vscode/src/query-history.ts
View File

@@ -205,13 +205,12 @@ export class HistoryTreeDataProvider extends DisposableObject {
? h2.initialInfo.start.getTime()
: h2.remoteQuery?.executionStartTime;
// result count for remote queries is not available here.
const resultCount1 = h1.t === 'local'
? h1.completedQuery?.resultCount ?? -1
: -1;
: h1.resultCount ?? -1;
const resultCount2 = h2.t === 'local'
? h2.completedQuery?.resultCount ?? -1
: -1;
: h2.resultCount ?? -1;
switch (this.sortOrder) {
case SortOrder.NameAsc:
@@ -505,7 +504,7 @@ export class QueryHistoryManager extends DisposableObject {
this.push(
queryHistoryConfigListener.onDidChangeConfiguration(() => {
this.treeDataProvider.refresh();
this.registerQueryHistoryScrubber(queryHistoryConfigListener, ctx);
this.registerQueryHistoryScrubber(queryHistoryConfigListener, this, ctx);
})
);
@@ -524,7 +523,7 @@ export class QueryHistoryManager extends DisposableObject {
},
}));
this.registerQueryHistoryScrubber(queryHistoryConfigListener, ctx);
this.registerQueryHistoryScrubber(queryHistoryConfigListener, this, ctx);
this.registerToRemoteQueriesEvents();
}
@@ -535,7 +534,7 @@ export class QueryHistoryManager extends DisposableObject {
/**
* Register and create the history scrubber.
*/
private registerQueryHistoryScrubber(queryHistoryConfigListener: QueryHistoryConfig, ctx: ExtensionContext) {
private registerQueryHistoryScrubber(queryHistoryConfigListener: QueryHistoryConfig, qhm: QueryHistoryManager, ctx: ExtensionContext) {
this.queryHistoryScrubber?.dispose();
// Every hour check if we need to re-run the query history scrubber.
this.queryHistoryScrubber = this.push(
@@ -544,6 +543,7 @@ export class QueryHistoryManager extends DisposableObject {
TWO_HOURS_IN_MS,
queryHistoryConfigListener.ttlInMillis,
this.queryStorageDir,
qhm,
ctx
)
);
@@ -573,6 +573,10 @@ export class QueryHistoryManager extends DisposableObject {
const remoteQueryHistoryItem = item as RemoteQueryHistoryItem;
remoteQueryHistoryItem.status = event.status;
remoteQueryHistoryItem.failureReason = event.failureReason;
remoteQueryHistoryItem.resultCount = event.resultCount;
if (event.status === QueryStatus.Completed) {
remoteQueryHistoryItem.completed = true;
}
await this.refreshTreeView();
} else {
void logger.log('Variant analysis status update event received for unknown variant analysis');
@@ -639,6 +643,15 @@ export class QueryHistoryManager extends DisposableObject {
return this.treeDataProvider.getCurrent();
}
async removeDeletedQueries() {
await Promise.all(this.treeDataProvider.allHistory.map(async (item) => {
if (item.t == 'local' && item.completedQuery && !(await fs.pathExists(item.completedQuery?.query.querySaveDir))) {
this.treeDataProvider.remove(item);
item.completedQuery?.dispose();
}
}));
}
async handleRemoveHistoryItem(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[] = []

8
extensions/ql-vscode/src/query-serialization.ts
View File

@@ -4,6 +4,7 @@ import * as path from 'path';
import { showAndLogErrorMessage } from './helpers';
import { asyncFilter, getErrorMessage, getErrorStack } from './pure/helpers-pure';
import { CompletedQueryInfo, LocalQueryInfo, QueryHistoryInfo } from './query-results';
import { QueryStatus } from './query-status';
import { QueryEvaluationInfo } from './run-queries';
export async function slurpQueryHistory(fsPath: string): Promise<QueryHistoryInfo[]> {
@@ -39,7 +40,12 @@ export async function slurpQueryHistory(fsPath: string): Promise<QueryHistoryInf
q.completedQuery.dispose = () => { /**/ };
}
} else if (q.t === 'remote') {
// noop
// A bug was introduced that didn't set the completed flag in query history
// items. The following code makes sure that the flag is set in order to
// "patch" older query history items.
if (q.status === QueryStatus.Completed) {
q.completed = true;
}
}
return q;
});

4
extensions/ql-vscode/src/queryserver-client.ts
View File

@@ -267,6 +267,10 @@ export function findQueryEvalLogSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary');
}
export function findJsonQueryEvalLogSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary.jsonl');
}
export function findQueryEvalLogEndSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log-end.summary');
}

22
extensions/ql-vscode/src/remote-queries/export-results.ts
View File

@@ -4,14 +4,17 @@ import * as fs from 'fs-extra';
import { window, commands, Uri, ExtensionContext, QuickPickItem, workspace, ViewColumn } from 'vscode';
import { Credentials } from '../authentication';
import { UserCancellationException } from '../commandRunner';
import { showInformationMessageWithAction } from '../helpers';
import {
showInformationMessageWithAction,
pluralize
} from '../helpers';
import { logger } from '../logging';
import { QueryHistoryManager } from '../query-history';
import { createGist } from './gh-actions-api-client';
import { RemoteQueriesManager } from './remote-queries-manager';
import { generateMarkdown } from './remote-queries-markdown-generation';
import { RemoteQuery } from './remote-query';
import { AnalysisResults } from './shared/analysis-result';
import { AnalysisResults, sumAnalysesResults } from './shared/analysis-result';
/**
* Exports the results of the currently-selected remote query.
@@ -74,13 +77,13 @@ async function determineExportFormat(
/**
* Converts the results of a remote query to markdown and uploads the files as a secret gist.
*/
async function exportResultsToGist(
export async function exportResultsToGist(
ctx: ExtensionContext,
query: RemoteQuery,
analysesResults: AnalysisResults[]
): Promise<void> {
const credentials = await Credentials.initialize(ctx);
const description = 'CodeQL Variant Analysis Results';
const description = buildGistDescription(query, analysesResults);
const markdownFiles = generateMarkdown(query, analysesResults, 'gist');
// Convert markdownFiles to the appropriate format for uploading to gist
const gistFiles = markdownFiles.reduce((acc, cur) => {
@@ -100,6 +103,17 @@ async function exportResultsToGist(
}
}
/**
* Builds Gist description
* Ex: Empty Block (Go) x results (y repositories)
*/
const buildGistDescription = (query: RemoteQuery, analysesResults: AnalysisResults[]) => {
const resultCount = sumAnalysesResults(analysesResults);
const resultLabel = pluralize(resultCount, 'result', 'results');
const repositoryLabel = pluralize(query.repositoryCount, 'repository', 'repositories');
return `${query.queryName} (${query.language}) ${resultLabel} (${repositoryLabel})`;
};
/**
* Converts the results of a remote query to markdown and saves the files locally
* in the query directory (where query results and metadata are also saved).

23
extensions/ql-vscode/src/remote-queries/remote-queries-interface.ts
View File

@@ -18,10 +18,16 @@ import {
import { Logger } from '../logging';
import { getHtmlForWebview } from '../interface-utils';
import { assertNever } from '../pure/helpers-pure';
import { AnalysisSummary, RemoteQueryResult } from './remote-query-result';
import {
AnalysisSummary,
RemoteQueryResult,
sumAnalysisSummariesResults
} from './remote-query-result';
import { RemoteQuery } from './remote-query';
import { RemoteQueryResult as RemoteQueryResultViewModel } from './shared/remote-query-result';
import { AnalysisSummary as AnalysisResultViewModel } from './shared/remote-query-result';
import {
AnalysisSummary as AnalysisResultViewModel,
RemoteQueryResult as RemoteQueryResultViewModel
} from './shared/remote-query-result';
import { showAndLogWarningMessage } from '../helpers';
import { URLSearchParams } from 'url';
import { SHOW_QUERY_TEXT_MSG } from '../query-history';
@@ -73,7 +79,7 @@ export class RemoteQueriesInterfaceManager {
*/
private buildViewModel(query: RemoteQuery, queryResult: RemoteQueryResult): RemoteQueryResultViewModel {
const queryFileName = path.basename(query.queryFilePath);
const totalResultCount = queryResult.analysisSummaries.reduce((acc, cur) => acc + cur.resultCount, 0);
const totalResultCount = sumAnalysisSummariesResults(queryResult.analysisSummaries);
const executionDuration = this.getDuration(queryResult.executionEndTime, query.executionStartTime);
const analysisSummaries = this.buildAnalysisSummaries(queryResult.analysisSummaries);
const totalRepositoryCount = queryResult.analysisSummaries.length;
@@ -111,6 +117,7 @@ export class RemoteQueriesInterfaceManager {
localResourceRoots: [
Uri.file(this.analysesResultsManager.storagePath),
Uri.file(path.join(this.ctx.extensionPath, 'out')),
Uri.file(path.join(this.ctx.extensionPath, 'node_modules/@vscode/codicons/dist')),
],
}
));
@@ -135,10 +142,16 @@ export class RemoteQueriesInterfaceManager {
ctx.asAbsolutePath('out/remote-queries/view/remoteQueries.css')
);
// Allows use of the VS Code "codicons" icon set.
// See https://github.com/microsoft/vscode-codicons
const codiconsPathOnDisk = Uri.file(
ctx.asAbsolutePath('node_modules/@vscode/codicons/dist/codicon.css')
);
panel.webview.html = getHtmlForWebview(
panel.webview,
scriptPathOnDisk,
[baseStylesheetUriOnDisk, stylesheetPathOnDisk],
[baseStylesheetUriOnDisk, stylesheetPathOnDisk, codiconsPathOnDisk],
true
);
ctx.subscriptions.push(

14
extensions/ql-vscode/src/remote-queries/remote-queries-manager.ts
View File

@@ -15,7 +15,7 @@ import { RemoteQuery } from './remote-query';
import { RemoteQueriesMonitor } from './remote-queries-monitor';
import { getRemoteQueryIndex, getRepositoriesMetadata, RepositoriesMetadata } from './gh-actions-api-client';
import { RemoteQueryResultIndex } from './remote-query-result-index';
import { RemoteQueryResult } from './remote-query-result';
import { RemoteQueryResult, sumAnalysisSummariesResults } from './remote-query-result';
import { DownloadLink } from './download-link';
import { AnalysesResultsManager } from './analyses-results-manager';
import { assertNever } from '../pure/helpers-pure';
@@ -41,6 +41,8 @@ export interface UpdatedQueryStatusEvent {
queryId: string;
status: QueryStatus;
failureReason?: string;
repositoryCount?: number;
resultCount?: number;
}
export class RemoteQueriesManager extends DisposableObject {
@@ -248,7 +250,7 @@ export class RemoteQueriesManager extends DisposableObject {
}
private async askToOpenResults(query: RemoteQuery, queryResult: RemoteQueryResult): Promise<void> {
const totalResultCount = queryResult.analysisSummaries.reduce((acc, cur) => acc + cur.resultCount, 0);
const totalResultCount = sumAnalysisSummariesResults(queryResult.analysisSummaries);
const totalRepoCount = queryResult.analysisSummaries.length;
const message = `Query "${query.queryName}" run on ${totalRepoCount} repositories and returned ${totalResultCount} results`;
@@ -314,9 +316,15 @@ export class RemoteQueriesManager extends DisposableObject {
): Promise<void> {
const resultIndex = await getRemoteQueryIndex(credentials, remoteQuery);
if (resultIndex) {
this.remoteQueryStatusUpdateEventEmitter.fire({ queryId, status: QueryStatus.Completed });
const metadata = await this.getRepositoriesMetadata(resultIndex, credentials);
const queryResult = this.mapQueryResult(executionEndTime, resultIndex, queryId, metadata);
const resultCount = sumAnalysisSummariesResults(queryResult.analysisSummaries);
this.remoteQueryStatusUpdateEventEmitter.fire({
queryId,
status: QueryStatus.Completed,
repositoryCount: queryResult.analysisSummaries.length,
resultCount
});
await this.storeJsonFile(queryId, 'query-result.json', queryResult);

1
extensions/ql-vscode/src/remote-queries/remote-query-history-item.ts
View File

@@ -7,6 +7,7 @@ import { RemoteQuery } from './remote-query';
export interface RemoteQueryHistoryItem {
readonly t: 'remote';
failureReason?: string;
resultCount?: number;
status: QueryStatus;
completed: boolean;
readonly queryId: string,

7
extensions/ql-vscode/src/remote-queries/remote-query-result.ts
View File

@@ -18,3 +18,10 @@ export interface AnalysisSummary {
starCount?: number,
lastUpdated?: number,
}
/**
* Sums up the number of results for all repos queried via a remote query.
*/
export const sumAnalysisSummariesResults = (analysisSummaries: AnalysisSummary[]): number => {
return analysisSummaries.reduce((acc, cur) => acc + cur.resultCount, 0);
};

1
extensions/ql-vscode/src/remote-queries/remote-query.ts
View File

@@ -8,4 +8,5 @@ export interface RemoteQuery {
controllerRepository: Repository;
executionStartTime: number; // Use number here since it needs to be serialized and desserialized.
actionsWorkflowRunId: number;
repositoryCount: number;
}

43
extensions/ql-vscode/src/remote-queries/run-remote-query.ts
View File

@@ -11,6 +11,7 @@ import {
showAndLogErrorMessage,
showAndLogInformationMessage,
tryGetQueryMetadata,
pluralize,
tmpDir
} from '../helpers';
import { Credentials } from '../authentication';
@@ -258,17 +259,19 @@ export async function runRemoteQuery(
});
const actionBranch = getActionBranch();
const workflowRunId = await runRemoteQueriesApiRequest(credentials, actionBranch, language, repoSelection, owner, repo, base64Pack, dryRun);
const apiResponse = await runRemoteQueriesApiRequest(credentials, actionBranch, language, repoSelection, owner, repo, base64Pack, dryRun);
const queryStartTime = Date.now();
const queryMetadata = await tryGetQueryMetadata(cliServer, queryFile);
if (dryRun) {
return { queryDirPath: remoteQueryDir.path };
} else {
if (!workflowRunId) {
if (!apiResponse) {
return;
}
const workflowRunId = apiResponse.workflow_run_id;
const repositoryCount = apiResponse.repositories_queried.length;
const remoteQuery = await buildRemoteQueryEntity(
queryFile,
queryMetadata,
@@ -276,7 +279,8 @@ export async function runRemoteQuery(
repo,
queryStartTime,
workflowRunId,
language);
language,
repositoryCount);
// don't return the path because it has been deleted
return { query: remoteQuery };
@@ -301,7 +305,7 @@ async function runRemoteQueriesApiRequest(
repo: string,
queryPackBase64: string,
dryRun = false
): Promise<void | number> {
): Promise<void | QueriesResponse> {
const data = {
ref,
language,
@@ -336,7 +340,7 @@ async function runRemoteQueriesApiRequest(
);
const { popupMessage, logMessage } = parseResponse(owner, repo, response.data);
void showAndLogInformationMessage(popupMessage, { fullMessage: logMessage });
return response.data.workflow_run_id;
return response.data;
} catch (error: any) {
if (error.status === 404) {
void showAndLogErrorMessage(`Controller repository was not found. Please make sure it's a valid repo name.${eol}`);
@@ -349,44 +353,37 @@ async function runRemoteQueriesApiRequest(
const eol = os.EOL;
const eol2 = os.EOL + os.EOL;
/**
* Returns "N repository" if N is one, "N repositories" otherwise.
*/
function pluralizeRepositories(numRepositories: number) {
return `${numRepositories} ${numRepositories === 1 ? 'repository' : 'repositories'}`;
}
// exported for testing only
export function parseResponse(owner: string, repo: string, response: QueriesResponse) {
const repositoriesQueried = response.repositories_queried;
const numRepositoriesQueried = repositoriesQueried.length;
const repositoryCount = repositoriesQueried.length;
const popupMessage = `Successfully scheduled runs on ${pluralizeRepositories(numRepositoriesQueried)}. [Click here to see the progress](https://github.com/${owner}/${repo}/actions/runs/${response.workflow_run_id}).`
const popupMessage = `Successfully scheduled runs on ${pluralize(repositoryCount, 'repository', 'repositories')}. [Click here to see the progress](https://github.com/${owner}/${repo}/actions/runs/${response.workflow_run_id}).`
+ (response.errors ? `${eol2}Some repositories could not be scheduled. See extension log for details.` : '');
let logMessage = `Successfully scheduled runs on ${pluralizeRepositories(numRepositoriesQueried)}. See https://github.com/${owner}/${repo}/actions/runs/${response.workflow_run_id}.`;
let logMessage = `Successfully scheduled runs on ${pluralize(repositoryCount, 'repository', 'repositories')}. See https://github.com/${owner}/${repo}/actions/runs/${response.workflow_run_id}.`;
logMessage += `${eol2}Repositories queried:${eol}${repositoriesQueried.join(', ')}`;
if (response.errors) {
const { invalid_repositories, repositories_without_database, private_repositories, cutoff_repositories, cutoff_repositories_count } = response.errors;
logMessage += `${eol2}Some repositories could not be scheduled.`;
if (invalid_repositories?.length) {
logMessage += `${eol2}${pluralizeRepositories(invalid_repositories.length)} invalid and could not be found:${eol}${invalid_repositories.join(', ')}`;
logMessage += `${eol2}${pluralize(invalid_repositories.length, 'repository', 'repositories')} invalid and could not be found:${eol}${invalid_repositories.join(', ')}`;
}
if (repositories_without_database?.length) {
logMessage += `${eol2}${pluralizeRepositories(repositories_without_database.length)} did not have a CodeQL database available:${eol}${repositories_without_database.join(', ')}`;
logMessage += `${eol2}${pluralize(repositories_without_database.length, 'repository', 'repositories')} did not have a CodeQL database available:${eol}${repositories_without_database.join(', ')}`;
logMessage += `${eol}For each public repository that has not yet been added to the database service, we will try to create a database next time the store is updated.`;
}
if (private_repositories?.length) {
logMessage += `${eol2}${pluralizeRepositories(private_repositories.length)} not public:${eol}${private_repositories.join(', ')}`;
logMessage += `${eol2}${pluralize(private_repositories.length, 'repository', 'repositories')} not public:${eol}${private_repositories.join(', ')}`;
logMessage += `${eol}When using a public controller repository, only public repositories can be queried.`;
}
if (cutoff_repositories_count) {
logMessage += `${eol2}${pluralizeRepositories(cutoff_repositories_count)} over the limit for a single request`;
logMessage += `${eol2}${pluralize(cutoff_repositories_count, 'repository', 'repositories')} over the limit for a single request`;
if (cutoff_repositories) {
logMessage += `:${eol}${cutoff_repositories.join(', ')}`;
if (cutoff_repositories_count !== cutoff_repositories.length) {
const moreRepositories = cutoff_repositories_count - cutoff_repositories.length;
logMessage += `${eol}...${eol}And another ${pluralizeRepositories(moreRepositories)}.`;
logMessage += `${eol}...${eol}And another ${pluralize(moreRepositories, 'repository', 'repositories')}.`;
}
} else {
logMessage += '.';
@@ -432,7 +429,8 @@ async function buildRemoteQueryEntity(
controllerRepoName: string,
queryStartTime: number,
workflowRunId: number,
language: string
language: string,
repositoryCount: number
): Promise<RemoteQuery> {
// The query name is either the name as specified in the query metadata, or the file name.
const queryName = queryMetadata?.name ?? path.basename(queryFilePath);
@@ -449,6 +447,7 @@ async function buildRemoteQueryEntity(
name: controllerRepoName,
},
executionStartTime: queryStartTime,
actionsWorkflowRunId: workflowRunId
actionsWorkflowRunId: workflowRunId,
repositoryCount,
};
}

6
extensions/ql-vscode/src/remote-queries/shared/analysis-result.ts
View File

@@ -90,3 +90,9 @@ export const getAnalysisResultCount = (analysisResults: AnalysisResults): number
const rawResultCount = analysisResults.rawResults?.resultSet.rows.length || 0;
return analysisResults.interpretedResults.length + rawResultCount;
};
/**
* Returns the total number of results for an analysis by adding all individual repo results.
*/
export const sumAnalysesResults = (analysesResults: AnalysisResults[]) =>
analysesResults.reduce((acc, curr) => acc + getAnalysisResultCount(curr), 0);

43
extensions/ql-vscode/src/remote-queries/view/CodePaths.tsx
View File

@@ -1,8 +1,8 @@
import { TriangleDownIcon, XCircleIcon } from '@primer/octicons-react';
import { ActionList, ActionMenu, Button, Overlay } from '@primer/react';
import { VSCodeLink, VSCodeTag } from '@vscode/webview-ui-toolkit/react';
import { XCircleIcon } from '@primer/octicons-react';
import { Overlay } from '@primer/react';
import { VSCodeDropdown, VSCodeLink, VSCodeOption, VSCodeTag } from '@vscode/webview-ui-toolkit/react';
import * as React from 'react';
import { useRef, useState } from 'react';
import { ChangeEvent, useRef, useState } from 'react';
import styled from 'styled-components';
import { CodeFlow, AnalysisMessage, ResultSeverity } from '../shared/analysis-result';
import FileCodeSnippet from './FileCodeSnippet';
@@ -94,25 +94,22 @@ const Menu = ({
codeFlows: CodeFlow[],
setSelectedCodeFlow: (value: React.SetStateAction<CodeFlow>) => void
}) => {
return <ActionMenu>
<ActionMenu.Anchor>
<Button variant="invisible" sx={{ fontWeight: 'normal', color: 'var(--vscode-editor-foreground);', padding: 0 }} >
{getCodeFlowName(codeFlows[0])}
<TriangleDownIcon size={16} />
</Button>
</ActionMenu.Anchor>
<ActionMenu.Overlay sx={{ backgroundColor: 'var(--vscode-editor-background)' }}>
<ActionList>
{codeFlows.map((codeFlow, index) =>
<ActionList.Item
key={`codeflow-${index}'`}
onSelect={(e: React.MouseEvent) => { setSelectedCodeFlow(codeFlow); }}>
{getCodeFlowName(codeFlow)}
</ActionList.Item>
)}
</ActionList>
</ActionMenu.Overlay>
</ActionMenu>;
return <VSCodeDropdown
onChange={(event: ChangeEvent<HTMLSelectElement>) => {
const selectedOption = event.target;
const selectedIndex = selectedOption.value as unknown as number;
setSelectedCodeFlow(codeFlows[selectedIndex]);
}}
>
{codeFlows.map((codeFlow, index) =>
<VSCodeOption
key={`codeflow-${index}'`}
value={index}
>
{getCodeFlowName(codeFlow)}
</VSCodeOption>
)}
</VSCodeDropdown>;
};
const CodePaths = ({

24
extensions/ql-vscode/src/remote-queries/view/RepositoriesSearch.tsx
View File

@@ -1,7 +1,5 @@
import * as React from 'react';
import { ChangeEvent } from 'react';
import { TextInput } from '@primer/react';
import { SearchIcon } from '@primer/octicons-react';
import { VSCodeTextField } from '@vscode/webview-ui-toolkit/react';
interface RepositoriesSearchProps {
filterValue: string;
@@ -10,20 +8,16 @@ interface RepositoriesSearchProps {
const RepositoriesSearch = ({ filterValue, setFilterValue }: RepositoriesSearchProps) => {
return <>
<TextInput
block
sx={{
backgroundColor: 'var(--vscode-editor-background);',
color: 'var(--vscode-editor-foreground);',
width: 'calc(100% - 14px)',
}}
leadingVisual={SearchIcon}
aria-label="Repository search"
<VSCodeTextField
style={{ width: '100%' }}
placeholder='Filter by repository owner/name'
ariaLabel="Repository search"
name="repository-search"
placeholder="Filter by repository owner/name"
value={filterValue}
onChange={(e: ChangeEvent) => setFilterValue((e.target as HTMLInputElement).value)}
/>
onInput={(e: InputEvent) => setFilterValue((e.target as HTMLInputElement).value)}
>
<span slot="start" className="codicon codicon-search"></span>
</VSCodeTextField>
</>;
};

67
extensions/ql-vscode/src/run-queries.ts
View File

@@ -37,6 +37,7 @@ import { ensureMetadataIsComplete } from './query-results';
import { SELECT_QUERY_NAME } from './contextual/locationFinder';
import { DecodedBqrsChunk } from './pure/bqrs-cli-types';
import { getErrorMessage } from './pure/helpers-pure';
import { parseVisualizerData } from './pure/log-summary-parser';
/**
* run-queries.ts
@@ -103,6 +104,10 @@ export class QueryEvaluationInfo {
return qsClient.findQueryEvalLogSummaryFile(this.querySaveDir);
}
get jsonEvalLogSummaryPath() {
return qsClient.findJsonQueryEvalLogSummaryFile(this.querySaveDir);
}
get evalLogEndSummaryPath() {
return qsClient.findQueryEvalLogEndSummaryFile(this.querySaveDir);
}
@@ -198,22 +203,10 @@ export class QueryEvaluationInfo {
logPath: this.evalLogPath,
});
if (await this.hasEvalLog()) {
queryInfo.evalLogLocation = this.evalLogPath;
void qs.cliServer.generateLogSummary(this.evalLogPath, this.evalLogSummaryPath, this.evalLogEndSummaryPath)
.then(() => {
queryInfo.evalLogSummaryLocation = this.evalLogSummaryPath;
fs.readFile(this.evalLogEndSummaryPath, (err, buffer) => {
if (err) {
throw new Error(`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`);
}
void qs.logger.log(' --- Evaluator Log Summary --- ', { additionalLogLocation: this.logPath });
void qs.logger.log(buffer.toString(), { additionalLogLocation: this.logPath });
});
})
.catch(err => {
void showAndLogWarningMessage(`Failed to generate structured evaluator log summary. Reason: ${err.message}`);
});
this.displayHumanReadableLogSummary(queryInfo, qs);
if (config.isCanary()) {
this.parseJsonLogSummary(qs.cliServer);
}
} else {
void showAndLogWarningMessage(`Failed to write structured evaluator log to ${this.evalLogPath}.`);
}
@@ -339,6 +332,48 @@ export class QueryEvaluationInfo {
return fs.pathExists(this.evalLogPath);
}
/**
* Calls the appropriate CLI command to generate a human-readable log summary
* and logs to the Query Server console and query log file.
*/
displayHumanReadableLogSummary(queryInfo: LocalQueryInfo, qs: qsClient.QueryServerClient): void {
queryInfo.evalLogLocation = this.evalLogPath;
void qs.cliServer.generateLogSummary(this.evalLogPath, this.evalLogSummaryPath, this.evalLogEndSummaryPath)
.then(() => {
queryInfo.evalLogSummaryLocation = this.evalLogSummaryPath;
fs.readFile(this.evalLogEndSummaryPath, (err, buffer) => {
if (err) {
throw new Error(`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`);
}
void qs.logger.log(' --- Evaluator Log Summary --- ', { additionalLogLocation: this.logPath });
void qs.logger.log(buffer.toString(), { additionalLogLocation: this.logPath });
});
})
.catch(err => {
void showAndLogWarningMessage(`Failed to generate human-readable structured evaluator log summary. Reason: ${err.message}`);
});
}
/**
* Calls the appropriate CLI command to generate a JSON log summary and parse it
* into the appropriate data model for the log visualizer.
*/
parseJsonLogSummary(cliServer: cli.CodeQLCliServer): void {
void cliServer.generateJsonLogSummary(this.evalLogPath, this.jsonEvalLogSummaryPath)
.then(() => {
// TODO(angelapwen): Stream the file in.
fs.readFile(this.jsonEvalLogSummaryPath, (err, buffer) => {
if (err) {
throw new Error(`Could not read structured evaluator log summary JSON file at ${this.jsonEvalLogSummaryPath}.`);
}
parseVisualizerData(buffer.toString()); // Eventually this return value will feed into the tree visualizer.
});
})
.catch(err => {
void showAndLogWarningMessage(`Failed to generate JSON structured evaluator log summary. Reason: ${err.message}`);
});
}
/**
* Creates the CSV file containing the results of this query. This will only be called if the query
* does not have interpreted results and the CSV file does not already exist.

462
extensions/ql-vscode/src/vscode-tests/no-workspace/data/remote-queries/query-with-results/analyses-results.json Normal file
View File

@@ -0,0 +1,462 @@
[
{
"nwo": "github/codeql",
"status": "Completed",
"interpretedResults": [
{
"message": {
"tokens": [
{
"t": "text",
"text": "This shell command depends on an uncontrolled "
},
{
"t": "location",
"text": "absolute path",
"location": {
"fileLink": {
"fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b",
"filePath": "javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js"
},
"highlightedRegion": {
"startLine": 4,
"startColumn": 35,
"endLine": 4,
"endColumn": 44
}
}
},
{ "t": "text", "text": "." }
]
},
"shortDescription": "This shell command depends on an uncontrolled ,absolute path,.",
"fileLink": {
"fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b",
"filePath": "javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js"
},
"severity": "Warning",
"codeSnippet": {
"startLine": 3,
"endLine": 6,
"text": "function cleanupTemp() {\n let cmd = \"rm -rf \" + path.join(__dirname, \"temp\");\n cp.execSync(cmd); // BAD\n}\n"
},
"highlightedRegion": {
"startLine": 5,
"startColumn": 15,
"endLine": 5,
"endColumn": 18
},
"codeFlows": [
{
"threadFlows": [
{
"fileLink": {
"fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b",
"filePath": "javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js"
},
"codeSnippet": {
"startLine": 2,
"endLine": 6,
"text": " path = require(\"path\");\nfunction cleanupTemp() {\n let cmd = \"rm -rf \" + path.join(__dirname, \"temp\");\n cp.execSync(cmd); // BAD\n}\n"
},
"highlightedRegion": {
"startLine": 4,
"startColumn": 35,
"endLine": 4,
"endColumn": 44
}
},
{
"fileLink": {
"fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b",
"filePath": "javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js"
},
"codeSnippet": {
"startLine": 2,
"endLine": 6,
"text": " path = require(\"path\");\nfunction cleanupTemp() {\n let cmd = \"rm -rf \" + path.join(__dirname, \"temp\");\n cp.execSync(cmd); // BAD\n}\n"
},
"highlightedRegion": {
"startLine": 4,
"startColumn": 25,
"endLine": 4,
"endColumn": 53
}
},
{
"fileLink": {
"fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b",
"filePath": "javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js"
},
"codeSnippet": {
"startLine": 2,
"endLine": 6,
"text": " path = require(\"path\");\nfunction cleanupTemp() {\n let cmd = \"rm -rf \" + path.join(__dirname, \"temp\");\n cp.execSync(cmd); // BAD\n}\n"
},
"highlightedRegion": {
"startLine": 4,
"startColumn": 13,
"endLine": 4,
"endColumn": 53
}
},
{
"fileLink": {
"fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b",
"filePath": "javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js"
},
"codeSnippet": {
"startLine": 2,
"endLine": 6,
"text": " path = require(\"path\");\nfunction cleanupTemp() {\n let cmd = \"rm -rf \" + path.join(__dirname, \"temp\");\n cp.execSync(cmd); // BAD\n}\n"
},
"highlightedRegion": {
"startLine": 4,
"startColumn": 7,
"endLine": 4,
"endColumn": 53
}
},
{
"fileLink": {
"fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b",
"filePath": "javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js"
},
"codeSnippet": {
"startLine": 3,
"endLine": 6,
"text": "function cleanupTemp() {\n let cmd = \"rm -rf \" + path.join(__dirname, \"temp\");\n cp.execSync(cmd); // BAD\n}\n"
},
"highlightedRegion": {
"startLine": 5,
"startColumn": 15,
"endLine": 5,
"endColumn": 18
}
}
]
}
]
},
{
"message": {
"tokens": [
{
"t": "text",
"text": "This shell command depends on an uncontrolled "
},
{
"t": "location",
"text": "absolute path",
"location": {
"fileLink": {
"fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b",
"filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js"
},
"highlightedRegion": {
"startLine": 6,
"startColumn": 36,
"endLine": 6,
"endColumn": 45
}
}
},
{ "t": "text", "text": "." }
]
},
"shortDescription": "This shell command depends on an uncontrolled ,absolute path,.",
"fileLink": {
"fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b",
"filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js"
},
"severity": "Warning",
"codeSnippet": {
"startLine": 4,
"endLine": 8,
"text": "(function() {\n\tcp.execFileSync('rm', ['-rf', path.join(__dirname, \"temp\")]); // GOOD\n\tcp.execSync('rm -rf ' + path.join(__dirname, \"temp\")); // BAD\n\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n"
},
"highlightedRegion": {
"startLine": 6,
"startColumn": 14,
"endLine": 6,
"endColumn": 54
},
"codeFlows": [
{
"threadFlows": [
{
"fileLink": {
"fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b",
"filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js"
},
"codeSnippet": {
"startLine": 4,
"endLine": 8,
"text": "(function() {\n\tcp.execFileSync('rm', ['-rf', path.join(__dirname, \"temp\")]); // GOOD\n\tcp.execSync('rm -rf ' + path.join(__dirname, \"temp\")); // BAD\n\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n"
},
"highlightedRegion": {
"startLine": 6,
"startColumn": 36,
"endLine": 6,
"endColumn": 45
}
},
{
"fileLink": {
"fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b",
"filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js"
},
"codeSnippet": {
"startLine": 4,
"endLine": 8,
"text": "(function() {\n\tcp.execFileSync('rm', ['-rf', path.join(__dirname, \"temp\")]); // GOOD\n\tcp.execSync('rm -rf ' + path.join(__dirname, \"temp\")); // BAD\n\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n"
},
"highlightedRegion": {
"startLine": 6,
"startColumn": 26,
"endLine": 6,
"endColumn": 54
}
},
{
"fileLink": {
"fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b",
"filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js"
},
"codeSnippet": {
"startLine": 4,
"endLine": 8,
"text": "(function() {\n\tcp.execFileSync('rm', ['-rf', path.join(__dirname, \"temp\")]); // GOOD\n\tcp.execSync('rm -rf ' + path.join(__dirname, \"temp\")); // BAD\n\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n"
},
"highlightedRegion": {
"startLine": 6,
"startColumn": 14,
"endLine": 6,
"endColumn": 54
}
}
]
}
]
}
]
},
{
"nwo": "test/no-results",
"status": "Completed",
"interpretedResults": []
},
{
"nwo": "meteor/meteor",
"status": "Completed",
"interpretedResults": [
{
"message": {
"tokens": [
{
"t": "text",
"text": "This shell command depends on an uncontrolled "
},
{
"t": "location",
"text": "absolute path",
"location": {
"fileLink": {
"fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec",
"filePath": "npm-packages/meteor-installer/config.js"
},
"highlightedRegion": {
"startLine": 39,
"startColumn": 20,
"endLine": 39,
"endColumn": 61
}
}
},
{ "t": "text", "text": "." }
]
},
"shortDescription": "This shell command depends on an uncontrolled ,absolute path,.",
"fileLink": {
"fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec",
"filePath": "npm-packages/meteor-installer/install.js"
},
"severity": "Warning",
"codeSnippet": {
"startLine": 257,
"endLine": 261,
"text": " if (isWindows()) {\n //set for the current session and beyond\n child_process.execSync(`setx path \"${meteorPath}/;%path%`);\n return;\n }\n"
},
"highlightedRegion": {
"startLine": 259,
"startColumn": 28,
"endLine": 259,
"endColumn": 62
},
"codeFlows": [
{
"threadFlows": [
{
"fileLink": {
"fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec",
"filePath": "npm-packages/meteor-installer/config.js"
},
"codeSnippet": {
"startLine": 37,
"endLine": 41,
"text": "\nconst meteorLocalFolder = '.meteor';\nconst meteorPath = path.resolve(rootPath, meteorLocalFolder);\n\nmodule.exports = {\n"
},
"highlightedRegion": {
"startLine": 39,
"startColumn": 20,
"endLine": 39,
"endColumn": 61
}
},
{
"fileLink": {
"fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec",
"filePath": "npm-packages/meteor-installer/config.js"
},
"codeSnippet": {
"startLine": 37,
"endLine": 41,
"text": "\nconst meteorLocalFolder = '.meteor';\nconst meteorPath = path.resolve(rootPath, meteorLocalFolder);\n\nmodule.exports = {\n"
},
"highlightedRegion": {
"startLine": 39,
"startColumn": 7,
"endLine": 39,
"endColumn": 61
}
},
{
"fileLink": {
"fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec",
"filePath": "npm-packages/meteor-installer/config.js"
},
"codeSnippet": {
"startLine": 42,
"endLine": 46,
"text": " METEOR_LATEST_VERSION,\n extractPath: rootPath,\n meteorPath,\n release: process.env.INSTALL_METEOR_VERSION || METEOR_LATEST_VERSION,\n rootPath,\n"
},
"highlightedRegion": {
"startLine": 44,
"startColumn": 3,
"endLine": 44,
"endColumn": 13
}
},
{
"fileLink": {
"fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec",
"filePath": "npm-packages/meteor-installer/install.js"
},
"codeSnippet": {
"startLine": 10,
"endLine": 14,
"text": "const os = require('os');\nconst {\n meteorPath,\n release,\n startedPath,\n"
},
"highlightedRegion": {
"startLine": 12,
"startColumn": 3,
"endLine": 12,
"endColumn": 13
}
},
{
"fileLink": {
"fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec",
"filePath": "npm-packages/meteor-installer/install.js"
},
"codeSnippet": {
"startLine": 9,
"endLine": 25,
"text": "const tmp = require('tmp');\nconst os = require('os');\nconst {\n meteorPath,\n release,\n startedPath,\n extractPath,\n isWindows,\n rootPath,\n sudoUser,\n isSudo,\n isMac,\n METEOR_LATEST_VERSION,\n shouldSetupExecPath,\n} = require('./config.js');\nconst { uninstall } = require('./uninstall');\nconst {\n"
},
"highlightedRegion": {
"startLine": 11,
"startColumn": 7,
"endLine": 23,
"endColumn": 27
}
},
{
"fileLink": {
"fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec",
"filePath": "npm-packages/meteor-installer/install.js"
},
"codeSnippet": {
"startLine": 257,
"endLine": 261,
"text": " if (isWindows()) {\n //set for the current session and beyond\n child_process.execSync(`setx path \"${meteorPath}/;%path%`);\n return;\n }\n"
},
"highlightedRegion": {
"startLine": 259,
"startColumn": 42,
"endLine": 259,
"endColumn": 52
}
},
{
"fileLink": {
"fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec",
"filePath": "npm-packages/meteor-installer/install.js"
},
"codeSnippet": {
"startLine": 257,
"endLine": 261,
"text": " if (isWindows()) {\n //set for the current session and beyond\n child_process.execSync(`setx path \"${meteorPath}/;%path%`);\n return;\n }\n"
},
"highlightedRegion": {
"startLine": 259,
"startColumn": 28,
"endLine": 259,
"endColumn": 62
}
}
]
},
{
"threadFlows": [
{
"fileLink": {
"fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec",
"filePath": "npm-packages/meteor-installer/config.js"
},
"codeSnippet": {
"startLine": 37,
"endLine": 41,
"text": "\nconst meteorLocalFolder = '.meteor';\nconst meteorPath = path.resolve(rootPath, meteorLocalFolder);\n\nmodule.exports = {\n"
},
"highlightedRegion": {
"startLine": 39,
"startColumn": 20,
"endLine": 39,
"endColumn": 61
}
},
{
"fileLink": {
"fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec",
"filePath": "npm-packages/meteor-installer/install.js"
},
"codeSnippet": {
"startLine": 257,
"endLine": 261,
"text": " if (isWindows()) {\n //set for the current session and beyond\n child_process.execSync(`setx path \"${meteorPath}/;%path%`);\n return;\n }\n"
},
"highlightedRegion": {
"startLine": 259,
"startColumn": 28,
"endLine": 259,
"endColumn": 62
}
}
]
}
]
}
]
}
]

10
extensions/ql-vscode/src/vscode-tests/no-workspace/data/remote-queries/query-with-results/query.json Normal file
View File

@@ -0,0 +1,10 @@
{
"queryName": "Shell command built from environment values",
"queryFilePath": "c:\\git-repo\\vscode-codeql-starter\\ql\\javascript\\ql\\src\\Security\\CWE-078\\ShellCommandInjectionFromEnvironment.ql",
"queryText": "/**\n * @name Shell command built from environment values\n * @description Building a shell command string with values from the enclosing\n * environment may cause subtle bugs or vulnerabilities.\n * @kind path-problem\n * @problem.severity warning\n * @security-severity 6.3\n * @precision high\n * @id js/shell-command-injection-from-environment\n * @tags correctness\n * security\n * external/cwe/cwe-078\n * external/cwe/cwe-088\n */\n\nimport javascript\nimport DataFlow::PathGraph\nimport semmle.javascript.security.dataflow.ShellCommandInjectionFromEnvironmentQuery\n\nfrom\n Configuration cfg, DataFlow::PathNode source, DataFlow::PathNode sink, DataFlow::Node highlight,\n Source sourceNode\nwhere\n sourceNode = source.getNode() and\n cfg.hasFlowPath(source, sink) and\n if cfg.isSinkWithHighlight(sink.getNode(), _)\n then cfg.isSinkWithHighlight(sink.getNode(), highlight)\n else highlight = sink.getNode()\nselect highlight, source, sink, \"This shell command depends on an uncontrolled $@.\", sourceNode,\n sourceNode.getSourceType()\n",
"language": "javascript",
"controllerRepository": { "owner": "dsp-testing", "name": "qc-controller" },
"executionStartTime": 1649419081990,
"actionsWorkflowRunId": 2115000864,
"repositoryCount": 10
}

32
extensions/ql-vscode/src/vscode-tests/no-workspace/history-item-label-provider.test.ts
View File

@@ -27,10 +27,10 @@ describe('HistoryItemLabelProvider', () => {
expect(labelProvider.getLabel(fqi)).to.eq('xxx');
fqi.userSpecifiedLabel = '%t %q %d %s %f %r %%';
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name db-name in progress query-file.ql 456 results %`);
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name db-name in progress query-file.ql (456 results) %`);
fqi.userSpecifiedLabel = '%t %q %d %s %f %r %%::%t %q %d %s %f %r %%';
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name db-name in progress query-file.ql 456 results %::${dateStr} query-name db-name in progress query-file.ql 456 results %`);
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name db-name in progress query-file.ql (456 results) %::${dateStr} query-name db-name in progress query-file.ql (456 results) %`);
});
it('should interpolate query when not user specified', () => {
@@ -40,10 +40,10 @@ describe('HistoryItemLabelProvider', () => {
config.format = '%t %q %d %s %f %r %%';
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name db-name in progress query-file.ql 456 results %`);
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name db-name in progress query-file.ql (456 results) %`);
config.format = '%t %q %d %s %f %r %%::%t %q %d %s %f %r %%';
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name db-name in progress query-file.ql 456 results %::${dateStr} query-name db-name in progress query-file.ql 456 results %`);
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name db-name in progress query-file.ql (456 results) %::${dateStr} query-name db-name in progress query-file.ql (456 results) %`);
});
it('should get query short label', () => {
@@ -89,23 +89,30 @@ describe('HistoryItemLabelProvider', () => {
expect(labelProvider.getLabel(fqi)).to.eq('xxx');
fqi.userSpecifiedLabel = '%t %q %d %s %%';
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name github/vscode-codeql-integration-tests in progress %`);
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name (javascript) github/vscode-codeql-integration-tests in progress %`);
fqi.userSpecifiedLabel = '%t %q %d %s %%::%t %q %d %s %%';
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name github/vscode-codeql-integration-tests in progress %::${dateStr} query-name github/vscode-codeql-integration-tests in progress %`);
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name (javascript) github/vscode-codeql-integration-tests in progress %::${dateStr} query-name (javascript) github/vscode-codeql-integration-tests in progress %`);
});
it('should interpolate query when not user specified', () => {
const fqi = createMockRemoteQueryInfo();
expect(labelProvider.getLabel(fqi)).to.eq('xxx query-name xxx');
expect(labelProvider.getLabel(fqi)).to.eq('xxx query-name (javascript) xxx');
config.format = '%t %q %d %s %f %r %%';
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name github/vscode-codeql-integration-tests in progress query-file.ql %`);
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name (javascript) github/vscode-codeql-integration-tests in progress query-file.ql (16 results) %`);
config.format = '%t %q %d %s %f %r %%::%t %q %d %s %f %r %%';
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name github/vscode-codeql-integration-tests in progress query-file.ql %::${dateStr} query-name github/vscode-codeql-integration-tests in progress query-file.ql %`);
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name (javascript) github/vscode-codeql-integration-tests in progress query-file.ql (16 results) %::${dateStr} query-name (javascript) github/vscode-codeql-integration-tests in progress query-file.ql (16 results) %`);
});
it('should use number of repositories instead of controller repo if available', () => {
const fqi = createMockRemoteQueryInfo(undefined, 2);
config.format = '%t %q %d %s %f %r %%';
expect(labelProvider.getLabel(fqi)).to.eq(`${dateStr} query-name (javascript) 2 repositories in progress query-file.ql (16 results) %`);
});
it('should get query short label', () => {
@@ -119,7 +126,7 @@ describe('HistoryItemLabelProvider', () => {
expect(labelProvider.getShortLabel(fqi)).to.eq('query-name');
});
function createMockRemoteQueryInfo(userSpecifiedLabel?: string) {
function createMockRemoteQueryInfo(userSpecifiedLabel?: string, repositoryCount?: number) {
return {
t: 'remote',
userSpecifiedLabel,
@@ -130,9 +137,12 @@ describe('HistoryItemLabelProvider', () => {
controllerRepository: {
owner: 'github',
name: 'vscode-codeql-integration-tests'
}
},
language: 'javascript',
repositoryCount,
},
status: 'in progress',
resultCount: 16,
} as unknown as RemoteQueryHistoryItem;
}
});

17
extensions/ql-vscode/src/vscode-tests/no-workspace/index.ts
View File

@@ -4,6 +4,7 @@ import * as sinonChai from 'sinon-chai';
import * as chai from 'chai';
import * as chaiAsPromised from 'chai-as-promised';
import 'chai/register-should';
import { ExtensionContext } from 'vscode';
import { runTestsInDirectory } from '../index-template';
@@ -13,3 +14,19 @@ chai.use(sinonChai);
export function run(): Promise<void> {
return runTestsInDirectory(__dirname);
}
export function createMockExtensionContext(): ExtensionContext {
return {
globalState: {
_state: {
'telemetry-request-viewed': true
} as Record<string, any>,
get(key: string) {
return this._state[key];
},
update(key: string, val: any) {
this._state[key] = val;
}
}
} as any;
}

12
extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts
View File

@@ -456,11 +456,11 @@ describe('query-history', () => {
describe('getChildren', () => {
const history = [
item('a', 2, 'remote'),
item('a', 2, 'remote', 24),
item('b', 10, 'local', 20),
item('c', 5, 'local', 30),
item('d', 1, 'local', 25),
item('e', 6, 'remote'),
item('e', 6, 'remote', 5),
];
let treeDataProvider: HistoryTreeDataProvider;
@@ -503,7 +503,7 @@ describe('query-history', () => {
});
it('should get children for result count ascending', async () => {
const expected = [history[0], history[4], history[1], history[3], history[2]];
const expected = [history[4], history[1], history[0], history[3], history[2]];
treeDataProvider.sortOrder = SortOrder.CountAsc;
const children = await treeDataProvider.getChildren();
@@ -511,7 +511,7 @@ describe('query-history', () => {
});
it('should get children for result count descending', async () => {
const expected = [history[0], history[4], history[1], history[3], history[2]].reverse();
const expected = [history[4], history[1], history[0], history[3], history[2]].reverse();
treeDataProvider.sortOrder = SortOrder.CountDesc;
const children = await treeDataProvider.getChildren();
@@ -573,6 +573,7 @@ describe('query-history', () => {
},
repositories: []
},
resultCount,
t
};
}
@@ -762,6 +763,9 @@ describe('query-history', () => {
TWO_HOURS_IN_MS,
LESS_THAN_ONE_DAY,
dir,
{
removeDeletedQueries: () => { return Promise.resolve(); }
} as QueryHistoryManager,
mockCtx,
{
increment: () => runCount++

56
extensions/ql-vscode/src/vscode-tests/no-workspace/remote-queries/export-results.test.ts Normal file
View File

@@ -0,0 +1,56 @@
import { expect } from 'chai';
import * as path from 'path';
import * as fs from 'fs-extra';
import * as sinon from 'sinon';
import * as pq from 'proxyquire';
import { ExtensionContext } from 'vscode';
import { createMockExtensionContext } from '../index';
import { Credentials } from '../../../authentication';
import { MarkdownFile } from '../../../remote-queries/remote-queries-markdown-generation';
import * as actionsApiClient from '../../../remote-queries/gh-actions-api-client';
import { exportResultsToGist } from '../../../remote-queries/export-results';
const proxyquire = pq.noPreserveCache();
describe('export results', async function() {
describe('exportResultsToGist', async function() {
let sandbox: sinon.SinonSandbox;
let mockCredentials: Credentials;
let mockResponse: sinon.SinonStub<any, Promise<{ status: number }>>;
let mockCreateGist: sinon.SinonStub;
let ctx: ExtensionContext;
beforeEach(() => {
sandbox = sinon.createSandbox();
mockCredentials = {
getOctokit: () => Promise.resolve({
request: mockResponse
})
} as unknown as Credentials;
sandbox.stub(Credentials, 'initialize').resolves(mockCredentials);
const resultFiles = [] as MarkdownFile[];
proxyquire('../../../remote-queries/remote-queries-markdown-generation', {
'generateMarkdown': sinon.stub().returns(resultFiles)
});
});
afterEach(() => {
sandbox.restore();
});
it('should call the GitHub Actions API with the correct gist title', async function() {
mockCreateGist = sinon.stub(actionsApiClient, 'createGist');
ctx = createMockExtensionContext();
const query = JSON.parse(await fs.readFile(path.join(__dirname, '../data/remote-queries/query-with-results/query.json'), 'utf8'));
const analysesResults = JSON.parse(await fs.readFile(path.join(__dirname, '../data/remote-queries/query-with-results/analyses-results.json'), 'utf8'));
await exportResultsToGist(ctx, query, analysesResults);
expect(mockCreateGist.calledOnce).to.be.true;
expect(mockCreateGist.firstCall.args[1]).to.equal('Shell command built from environment values (javascript) 3 results (10 repositories)');
});
});
});

17
extensions/ql-vscode/src/vscode-tests/no-workspace/telemetry.test.ts
View File

@@ -6,6 +6,7 @@ import { TelemetryListener, telemetryListener as globalTelemetryListener } from
import { UserCancellationException } from '../../commandRunner';
import { fail } from 'assert';
import { ENABLE_TELEMETRY } from '../../config';
import { createMockExtensionContext } from './index';
const sandbox = sinon.createSandbox();
@@ -339,22 +340,6 @@ describe('telemetry reporting', function() {
expect(window.showInformationMessage).to.have.been.calledOnce;
});
function createMockExtensionContext(): ExtensionContext {
return {
globalState: {
_state: {
'telemetry-request-viewed': true
} as Record<string, any>,
get(key: string) {
return this._state[key];
},
update(key: string, val: any) {
this._state[key] = val;
}
}
} as any;
}
async function enableTelemetry(section: string, value: boolean | undefined) {
await workspace.getConfiguration(section).update(
'enableTelemetry', value, ConfigurationTarget.Global

5
extensions/ql-vscode/test/pure-tests/evaluator-log-summaries/invalid-header.jsonl Normal file
View File

@@ -0,0 +1,5 @@
{
"summaryLogVersion" : "0.3.0",
"codeqlVersion" : "2.9.0+202204201304plus",
"startTime" : "2022-06-23T14:02:41.607Z"
}

14
extensions/ql-vscode/test/pure-tests/evaluator-log-summaries/invalid-summary.jsonl Normal file
View File

@@ -0,0 +1,14 @@
{
"completionTime" : "2022-06-23T14:02:42.019Z",
"raHash" : "8caddafufc3it9svjph9m8d43me",
"predicateName" : "function_instantiation",
"appearsAs" : {
"function_instantiation" : {
"uboot-taint.ql" : [ 3, 9, 10, 11, 13, 14, 15, 17, 18, 20, 21, 22, 23, 24 ]
}
},
"queryCausingWork" : "uboot-taint.ql",
"evaluationStrategy" : "EXTENSIONAL",
"millis" : 0,
"resultSize" : 0
}

124
extensions/ql-vscode/test/pure-tests/evaluator-log-summaries/valid-summary.jsonl Normal file
View File

@@ -0,0 +1,124 @@
{
"completionTime" : "2022-06-23T14:02:42.007Z",
"raHash" : "e77dfaa9ciimqv7gb3imoesdb11",
"predicateName" : "query::ClassPointerType::getBaseType#dispred#f0820431#ff",
"appearsAs" : {
"query::ClassPointerType::getBaseType#dispred#f0820431#ff" : {
"uboot-taint.ql" : [ 3 ]
}
},
"queryCausingWork" : "uboot-taint.ql",
"evaluationStrategy" : "COMPUTE_SIMPLE",
"millis" : 11,
"resultSize" : 1413,
"dependencies" : {
"derivedtypes_2013#join_rhs" : "0da8f1fqbiin9i0mcitjedvlbc7",
"query::Class#class#f0820431#f" : "2e5d24rnudi1l99mvtse9u567b7",
"derivedtypes" : "070e120iu2i3dhj6pt13oqnbi66"
},
"ra" : {
"pipeline" : [
" {1} r1 = CONSTANT(unique int)[1]",
" {3} r2 = JOIN r1 WITH derivedtypes_2013#join_rhs ON FIRST 1 OUTPUT Rhs.3, Rhs.1, Rhs.2",
" {4} r3 = JOIN r2 WITH query::Class#class#f0820431#f ON FIRST 1 OUTPUT Lhs.1, Lhs.2, 1, Lhs.0",
" {2} r4 = JOIN r3 WITH derivedtypes ON FIRST 4 OUTPUT Lhs.0, Lhs.3",
" return r4"
]
},
"pipelineRuns" : [ {
"raReference" : "pipeline"
} ]
}
{
"completionTime" : "2022-06-23T14:02:42.072Z",
"raHash" : "0a0e3cicgtsru1m0cun896qhi61",
"predicateName" : "query::DefinedMemberFunction#class#f0820431#f",
"appearsAs" : {
"query::DefinedMemberFunction#class#f0820431#f" : {
"uboot-taint.ql" : [ 3 ]
}
},
"queryCausingWork" : "uboot-taint.ql",
"evaluationStrategy" : "COMPUTE_SIMPLE",
"millis" : 20,
"resultSize" : 8740,
"dependencies" : {
"fun_decls" : "e6e2e6vn02fcrrtp1ks1f75cd01",
"fun_def" : "6bec314lcq8sm3skg9mpecscp02",
"function_instantiation" : "8caddafufc3it9svjph9m8d43me",
"fun_decls_10#join_rhs" : "16474d6lehg7ssk83gnv2q7r8t8"
},
"ra" : {
"pipeline" : [
" {2} r1 = SCAN fun_decls OUTPUT In.0, In.1",
" {2} r2 = STREAM DEDUP r1",
" {1} r3 = JOIN r2 WITH fun_def ON FIRST 1 OUTPUT Lhs.1",
"",
" {2} r4 = SCAN function_instantiation OUTPUT In.1, In.0",
" {2} r5 = JOIN r4 WITH fun_decls_10#join_rhs ON FIRST 1 OUTPUT Rhs.1, Lhs.1",
" {1} r6 = JOIN r5 WITH fun_def ON FIRST 1 OUTPUT Lhs.1",
"",
" {1} r7 = r3 UNION r6",
" return r7"
]
},
"pipelineRuns" : [ {
"raReference" : "pipeline"
} ]
}
{
"completionTime" : "2022-06-23T14:02:43.799Z",
"raHash" : "1d0c6wplpr6bnlnii51r7f6lh85",
"predicateName" : "QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff",
"appearsAs" : {
"QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff" : {
"uboot-taint.ql" : [ 10 ]
}
},
"queryCausingWork" : "uboot-taint.ql",
"evaluationStrategy" : "COMPUTE_RECURSIVE",
"millis" : 2,
"predicateIterationMillis" : [ 1, 0 ],
"deltaSizes" : [ 1, 0 ],
"resultSize" : 1,
"dependencies" : {
"namespaces" : "bf72dcmerq68kur5k2uttmjoco4",
"namespacembrs_1#antijoin_rhs" : "3c47bbkgae024k8hgf148mgeqi2",
"QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#join_rhs#1" : "d55d698lva8n15u4v7055ma1vl9",
"namespacembrs" : "08a148i70tonoa8fp0mb5eb44r1",
"QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#join_rhs" : "77cd9dha843p3v12qso4qpe4qoe"
},
"layerSize" : 1,
"ra" : {
"base" : [
" {2} r1 = namespaces AND NOT namespacembrs_1#antijoin_rhs(Lhs.0)",
" return r1"
],
"standard" : [
" {2} r1 = JOIN QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#prev_delta WITH QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#join_rhs#1 ON FIRST 1 OUTPUT Rhs.1, Lhs.1",
"",
" {2} r2 = JOIN QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#prev_delta WITH QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#join_rhs ON FIRST 1 OUTPUT Rhs.1, (Lhs.1 ++ \"::\" ++ Rhs.2)",
"",
" {2} r3 = r1 UNION r2",
" {2} r4 = r3 AND NOT QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#prev(Lhs.0, Lhs.1)",
" return r4"
],
"order_500000" : [
" {2} r1 = JOIN QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#prev_delta WITH QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#join_rhs#1 ON FIRST 1 OUTPUT Rhs.1, Lhs.1",
"",
" {2} r2 = JOIN QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#prev_delta WITH namespacembrs ON FIRST 1 OUTPUT Rhs.1, Lhs.1",
" {2} r3 = JOIN r2 WITH namespaces ON FIRST 1 OUTPUT Lhs.0, (Lhs.1 ++ \"::\" ++ Rhs.1)",
"",
" {2} r4 = r1 UNION r3",
" {2} r5 = r4 AND NOT QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#prev(Lhs.0, Lhs.1)",
" return r5"
]
},
"pipelineRuns" : [ {
"raReference" : "base"
}, {
"raReference" : "order_500000"
} ]
}

42
extensions/ql-vscode/test/pure-tests/log-summary-parser.test.ts Normal file
View File

@@ -0,0 +1,42 @@
import { expect } from 'chai';
import * as fs from 'fs-extra';
import * as path from 'path';
import 'mocha';
import { parseVisualizerData } from '../../src/pure/log-summary-parser';
describe('Evaluator log summary tests', async function () {
describe('for a valid summary text', async function () {
it('should return only valid EvaluatorLogData objects', async function () {
const validSummaryText = await fs.readFile(path.join(__dirname, 'evaluator-log-summaries/valid-summary.jsonl'), 'utf8');
const logDataItems = parseVisualizerData(validSummaryText.toString());
expect(logDataItems).to.not.be.undefined;
expect (logDataItems.length).to.eq(3);
for (const item of logDataItems) {
expect(item.queryCausingWork).to.not.be.empty;
expect(item.predicateName).to.not.be.empty;
expect(item.millis).to.be.a('number');
expect(item.resultSize).to.be.a('number');
expect(item.ra).to.not.be.undefined;
expect(item.ra).to.not.be.empty;
for (const [pipeline, steps] of Object.entries(item.ra)) {
expect (pipeline).to.not.be.empty;
expect (steps).to.not.be.undefined;
expect (steps.length).to.be.greaterThan(0);
}
}
});
it('should not parse a summary header object', async function () {
const invalidHeaderText = await fs.readFile(path.join(__dirname, 'evaluator-log-summaries/invalid-header.jsonl'), 'utf8');
const logDataItems = parseVisualizerData(invalidHeaderText);
expect (logDataItems.length).to.eq(0);
});
it('should not parse a log event missing RA or millis fields', async function () {
const invalidSummaryText = await fs.readFile(path.join(__dirname, 'evaluator-log-summaries/invalid-summary.jsonl'), 'utf8');
const logDataItems = parseVisualizerData(invalidSummaryText);
expect (logDataItems.length).to.eq(0);
});
});
});