mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'main' into aeisenberg/codeSnippet-handling

Browse Source
This commit is contained in:
Andrew Eisenberg
2022-03-29 14:28:22 -07:00
parent 61e674e9f6 2d1726763f
commit 6fa0227a1e
18 changed files with 1433 additions and 413 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
.github/dependabot.yml vendored Normal file
View File

@@ -0,0 +1,20 @@
version: 2
updates:
- package-ecosystem: "npm"
directory: "extensions/ql-vscode"
schedule:
interval: "daily"
labels:
- "Update dependencies"
ignore:
- dependency-name: "*"
update-types: ["version-update:semver-minor", "version-update:semver-patch"]
- package-ecosystem: "github-actions"
directory: ".github"
schedule:
interval: "daily"
labels:
- "Update dependencies"
ignore:
- dependency-name: "*"
update-types: ["version-update:semver-minor", "version-update:semver-patch"]

3
extensions/ql-vscode/CHANGELOG.md
View File

@@ -3,6 +3,8 @@
## [UNRELEASED]
- Fix a bug where the AST viewer was not synchronizing its selected node when the editor selection changes. [#1230](https://github.com/github/vscode-codeql/pull/1230)
- Open the directory in the finder/explorer (instead of just highlighting it) when running the "Open query directory" command from the query history view. [#1235](https://github.com/github/vscode-codeql/pull/1235)
- Ensure query label in the query history view changes are persisted across restarts. [#1235](https://github.com/github/vscode-codeql/pull/1235)
## 1.6.1 - 17 March 2022
@@ -15,6 +17,7 @@ No user facing changes.
- Fix a bug where queries took a long time to run if there are no folders in the workspace. [#1157](https://github.com/github/vscode-codeql/pull/1157)
- [BREAKING CHANGE] The `codeQL.runningQueries.customLogDirectory` setting is deprecated and no longer has any function. Instead, all query log files will be stored in the query history directory, next to the query results. [#1178](https://github.com/github/vscode-codeql/pull/1178)
- Add a _Open query directory_ command for query items. This command opens the directory containing all artifacts for a query. [#1179](https://github.com/github/vscode-codeql/pull/1179)
- Add options to display evaluator logs for a given query run. Some information that was previously found in the query server output may now be found here. [#1186](https://github.com/github/vscode-codeql/pull/1186)
## 1.5.11 - 10 February 2022

16
extensions/ql-vscode/gulpfile.ts/index.ts
View File

@@ -1,8 +1,8 @@
import * as gulp from 'gulp';
import { compileTypeScript, watchTypeScript, copyViewCss, cleanOutput } from './typescript';
import { compileTypeScript, watchTypeScript, copyViewCss, cleanOutput, watchCss } from './typescript';
import { compileTextMateGrammar } from './textmate';
import { copyTestData } from './tests';
import { compileView } from './webpack';
import { compileView, watchView } from './webpack';
import { packageExtension } from './package';
import { injectAppInsightsKey } from './appInsights';
@@ -14,5 +14,15 @@ export const buildWithoutPackage =
)
);
export { cleanOutput, compileTextMateGrammar, watchTypeScript, compileTypeScript, copyTestData, injectAppInsightsKey };
export {
cleanOutput,
compileTextMateGrammar,
watchTypeScript,
watchView,
compileTypeScript,
copyTestData,
injectAppInsightsKey,
compileView,
watchCss
};
export default gulp.series(buildWithoutPackage, injectAppInsightsKey, packageExtension);

4
extensions/ql-vscode/gulpfile.ts/typescript.ts
View File

@@ -40,6 +40,10 @@ export function watchTypeScript() {
gulp.watch('src/**/*.ts', compileTypeScript);
}
export function watchCss() {
gulp.watch('src/**/*.css', copyViewCss);
}
/** Copy CSS files for the results view into the output directory. */
export function copyViewCss() {
return gulp.src('src/**/view/*.css')

31
extensions/ql-vscode/gulpfile.ts/webpack.ts
View File

@@ -2,7 +2,23 @@ import * as webpack from 'webpack';
import { config } from './webpack.config';
export function compileView(cb: (err?: Error) => void) {
webpack(config).run((error, stats) => {
doWebpack(config, true, cb);
}
export function watchView(cb: (err?: Error) => void) {
const watchConfig = {
...config,
watch: true,
watchOptions: {
aggregateTimeout: 200,
poll: 1000,
}
};
doWebpack(watchConfig, false, cb);
}
function doWebpack(internalConfig: webpack.Configuration, failOnError: boolean, cb: (err?: Error) => void) {
const resultCb = (error: Error | undefined, stats?: webpack.Stats) => {
if (error) {
cb(error);
}
@@ -20,11 +36,16 @@ export function compileView(cb: (err?: Error) => void) {
errors: true
}));
if (stats.hasErrors()) {
cb(new Error('Compilation errors detected.'));
return;
if (failOnError) {
cb(new Error('Compilation errors detected.'));
return;
} else {
console.error('Compilation errors detected.');
}
}
cb();
}
};
cb();
});
webpack(internalConfig, resultCb);
}

1323
extensions/ql-vscode/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

38
extensions/ql-vscode/package.json
View File

@@ -522,6 +522,14 @@
"command": "codeQLQueryHistory.openQueryDirectory",
"title": "Open query directory"
},
{
"command": "codeQLQueryHistory.showEvalLog",
"title": "Show Evaluator Log (Raw)"
},
{
"command": "codeQLQueryHistory.showEvalLogSummary",
"title": "Show Evaluator Log (Summary)"
},
{
"command": "codeQLQueryHistory.cancel",
"title": "Cancel"
@@ -725,6 +733,16 @@
"group": "9_qlCommands",
"when": "view == codeQLQueryHistory && !hasRemoteServer"
},
{
"command": "codeQLQueryHistory.showEvalLog",
"group": "9_qlCommands",
"when": "codeql.supportsEvalLog && (viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem)"
},
{
"command": "codeQLQueryHistory.showEvalLogSummary",
"group": "9_qlCommands",
"when": "codeql.supportsEvalLog && (viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem)"
},
{
"command": "codeQLQueryHistory.showQueryText",
"group": "9_qlCommands",
@@ -924,6 +942,14 @@
"command": "codeQLQueryHistory.showQueryLog",
"when": "false"
},
{
"command": "codeQLQueryHistory.showEvalLog",
"when": "false"
},
{
"command": "codeQLQueryHistory.showEvalLogSummary",
"when": "false"
},
{
"command": "codeQLQueryHistory.openQueryDirectory",
"when": "false"
@@ -1072,6 +1098,8 @@
"build": "gulp",
"watch": "npm-run-all -p watch:*",
"watch:extension": "tsc --watch",
"watch:webpack": "gulp watchView",
"watch:css": "gulp watchCss",
"test": "mocha --exit -r ts-node/register test/pure-tests/**/*.ts",
"preintegration": "rm -rf ./out/vscode-tests && gulp",
"integration": "node ./out/vscode-tests/run-integration-tests.js no-workspace,minimal-workspace",
@@ -1089,7 +1117,7 @@
"classnames": "~2.2.6",
"d3": "^6.3.1",
"d3-graphviz": "^2.6.1",
"fs-extra": "^9.0.1",
"fs-extra": "^10.0.1",
"glob-promise": "^3.4.0",
"js-yaml": "^3.14.0",
"minimist": "~1.2.6",
@@ -1119,9 +1147,9 @@
"@types/chai-as-promised": "~7.1.2",
"@types/child-process-promise": "^2.2.1",
"@types/classnames": "~2.2.9",
"@types/del": "^4.0.0",
"@types/d3": "^6.2.0",
"@types/d3-graphviz": "^2.6.6",
"@types/del": "^4.0.0",
"@types/fs-extra": "^9.0.6",
"@types/glob": "^7.1.1",
"@types/google-protobuf": "^3.2.7",
@@ -1147,7 +1175,7 @@
"@types/tmp": "^0.1.0",
"@types/unzipper": "~0.10.1",
"@types/vscode": "^1.59.0",
"@types/webpack": "^4.32.1",
"@types/webpack": "^5.28.0",
"@types/xml2js": "~0.4.4",
"@typescript-eslint/eslint-plugin": "^4.26.0",
"@typescript-eslint/parser": "^4.26.0",
@@ -1175,13 +1203,13 @@
"sinon": "~9.0.0",
"sinon-chai": "~3.5.0",
"style-loader": "~0.23.1",
"through2": "^3.0.1",
"through2": "^4.0.2",
"ts-loader": "^8.1.0",
"ts-node": "^8.3.0",
"ts-protoc-gen": "^0.9.0",
"typescript": "^4.5.5",
"typescript-formatter": "^7.2.2",
"vsce": "^1.65.0",
"vsce": "^2.7.0",
"vscode-test": "^1.4.0",
"webpack": "^5.28.0",
"webpack-cli": "^4.6.0"

32
extensions/ql-vscode/src/cli.ts
View File

@@ -8,7 +8,7 @@ import { Readable } from 'stream';
import { StringDecoder } from 'string_decoder';
import * as tk from 'tree-kill';
import { promisify } from 'util';
import { CancellationToken, Disposable, Uri } from 'vscode';
import { CancellationToken, commands, Disposable, Uri } from 'vscode';
import { BQRSInfo, DecodedBqrsChunk } from './pure/bqrs-cli-types';
import { CliConfig } from './config';
@@ -665,6 +665,23 @@ export class CodeQLCliServer implements Disposable {
return await this.runCodeQlCliCommand(['generate', 'query-help'], subcommandArgs, `Generating qhelp in markdown format at ${outputDirectory}`);
}
/**
* Generate a summary of an evaluation log.
* @param inputPath The path of an evaluation event log.
* @param outputPath The path to write a human-readable summary of it to.
*/
async generateLogSummary(
inputPath: string,
outputPath: string,
): Promise<string> {
const subcommandArgs = [
'--format=text',
inputPath,
outputPath
];
return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating log summary');
}
/**
* Gets the results from a bqrs.
* @param bqrsPath The path to the bqrs.
@@ -940,6 +957,10 @@ export class CodeQLCliServer implements Disposable {
public async getVersion() {
if (!this._version) {
this._version = await this.refreshVersion();
// this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
await commands.executeCommand(
'setContext', 'codeql.supportsEvalLog', await this.cliConstraints.supportsPerQueryEvalLog()
);
}
return this._version;
}
@@ -1256,6 +1277,11 @@ export class CliVersionConstraint {
*/
public static CLI_VERSION_WITH_STRUCTURED_EVAL_LOG = new SemVer('2.8.2');
/**
* CLI version that supports rotating structured logs to produce one per query.
*/
public static CLI_VERSION_WITH_PER_QUERY_EVAL_LOG = new SemVer('2.8.4');
constructor(private readonly cli: CodeQLCliServer) {
/**/
}
@@ -1315,4 +1341,8 @@ export class CliVersionConstraint {
async supportsStructuredEvalLog() {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_STRUCTURED_EVAL_LOG);
}
async supportsPerQueryEvalLog() {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG);
}
}

68
extensions/ql-vscode/src/databaseFetcher.ts
View File

@@ -86,15 +86,15 @@ export async function promptImportGithubDatabase(
maxStep: 2
});
const githubRepo = await window.showInputBox({
title: 'Enter a GitHub repository in the format <owner>/<repo> (e.g. github/codeql)',
placeHolder: '<owner>/<repo>',
title: 'Enter a GitHub repository URL or "name with owner" (e.g. https://github.com/github/codeql or github/codeql)',
placeHolder: 'https://github.com/<owner>/<repo> or <owner>/<repo>',
ignoreFocusOut: true,
});
if (!githubRepo) {
return;
}
if (!REPO_REGEX.test(githubRepo)) {
if (!looksLikeGithubRepo(githubRepo)) {
throw new Error(`Invalid GitHub repository: ${githubRepo}`);
}
@@ -465,18 +465,62 @@ export async function findDirWithFile(
return;
}
/**
* The URL pattern is https://github.com/{owner}/{name}/{subpages}.
*
* This function accepts any URL that matches the pattern above. It also accepts just the
* name with owner (NWO): `<owner>/<repo>`.
*
* @param githubRepo The GitHub repository URL or NWO
*
* @return true if this looks like a valid GitHub repository URL or NWO
*/
export function looksLikeGithubRepo(
githubRepo: string | undefined
): githubRepo is string {
if (!githubRepo) {
return false;
}
if (REPO_REGEX.test(githubRepo) || convertGitHubUrlToNwo(githubRepo)) {
return true;
}
return false;
}
/**
* Converts a GitHub repository URL to the corresponding NWO.
* @param githubUrl The GitHub repository URL
* @return The corresponding NWO, or undefined if the URL is not valid
*/
function convertGitHubUrlToNwo(githubUrl: string): string | undefined {
try {
const uri = Uri.parse(githubUrl, true);
if (uri.scheme !== 'https') {
return;
}
if (uri.authority !== 'github.com' && uri.authority !== 'www.github.com') {
return;
}
const paths = uri.path.split('/').filter((segment: string) => segment);
const nwo = `${paths[0]}/${paths[1]}`;
if (REPO_REGEX.test(nwo)) {
return nwo;
}
return;
} catch (e) {
// Ignore the error here, since we catch failures at a higher level.
// In particular: returning undefined leads to an error in 'promptImportGithubDatabase'.
return;
}
}
export async function convertGithubNwoToDatabaseUrl(
githubRepo: string,
credentials: Credentials,
progress: ProgressCallback): Promise<string | undefined> {
try {
// TODO: In future, we could accept GitHub URLs in addition to NWOs.
// Similar to "looksLikeLgtmUrl".
if (!REPO_REGEX.test(githubRepo)) {
throw new Error('Invalid repository format. Must be in the format <owner>/<repo> (e.g. github/codeql)');
}
const [owner, repo] = githubRepo.split('/');
const nwo = convertGitHubUrlToNwo(githubRepo) || githubRepo;
const [owner, repo] = nwo.split('/');
const octokit = await credentials.getOctokit();
const response = await octokit.request('GET /repos/:owner/:repo/code-scanning/codeql/databases', { owner, repo });
@@ -531,7 +575,7 @@ export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string
return false;
}
const paths = uri.path.split('/').filter((segment) => segment);
const paths = uri.path.split('/').filter((segment: string) => segment);
return paths.length >= 4 && paths[0] === 'projects';
} catch (e) {
return false;
@@ -604,7 +648,7 @@ export async function convertLgtmUrlToDatabaseUrl(
async function downloadLgtmProjectMetadata(lgtmUrl: string): Promise<any> {
const uri = Uri.parse(lgtmUrl, true);
const paths = ['api', 'v1.0'].concat(
uri.path.split('/').filter((segment) => segment)
uri.path.split('/').filter((segment: string) => segment)
).slice(0, 6);
const projectUrl = `https://lgtm.com/${paths.join('/')}`;
const projectResponse = await fetch(projectUrl);

2
extensions/ql-vscode/src/extension.ts
View File

@@ -535,6 +535,8 @@ async function activateWithInstalledDistribution(
queryStorageDir,
progress,
source.token,
undefined,
item,
);
item.completeThisQuery(completedQueryInfo);
await showResultsForCompletedQuery(item as CompletedLocalQueryInfo, WebviewReveal.NotForced);

59
extensions/ql-vscode/src/pure/messages.ts
View File

@@ -646,6 +646,35 @@ export interface ClearCacheParams {
*/
dryRun: boolean;
}
/**
* Parameters to start a new structured log
*/
export interface StartLogParams {
/**
* The dataset for which we want to start a new structured log
*/
db: Dataset;
/**
* The path where we want to place the new structured log
*/
logPath: string;
}
/**
* Parameters to terminate a structured log
*/
export interface EndLogParams {
/**
* The dataset for which we want to terminated the log
*/
db: Dataset;
/**
* The path of the log to terminate, will be a no-op if we aren't logging here
*/
logPath: string;
}
/**
* Parameters for trimming the cache of a dataset
*/
@@ -682,6 +711,26 @@ export interface ClearCacheResult {
deletionMessage: string;
}
/**
* The result of starting a new structured log.
*/
export interface StartLogResult {
/**
* A user friendly message saying what happened.
*/
outcomeMessage: string;
}
/**
* The result of terminating a structured.
*/
export interface EndLogResult {
/**
* A user friendly message saying what happened.
*/
outcomeMessage: string;
}
/**
* Parameters for running a set of queries
*/
@@ -1018,6 +1067,16 @@ export const compileUpgrade = new rpc.RequestType<WithProgressId<CompileUpgradeP
*/
export const compileUpgradeSequence = new rpc.RequestType<WithProgressId<CompileUpgradeSequenceParams>, CompileUpgradeSequenceResult, void, void>('compilation/compileUpgradeSequence');
/**
* Start a new structured log in the evaluator, terminating the previous one if it exists
*/
export const startLog = new rpc.RequestType<WithProgressId<StartLogParams>, StartLogResult, void, void>('evaluation/startLog');
/**
* Terminate a structured log in the evaluator. Is a no-op if we aren't logging to the given location
*/
export const endLog = new rpc.RequestType<WithProgressId<EndLogParams>, EndLogResult, void, void>('evaluation/endLog');
/**
* Clear the cache of a dataset
*/

131
extensions/ql-vscode/src/query-history.ts
View File

@@ -28,12 +28,14 @@ import { URLSearchParams } from 'url';
import { QueryServerClient } from './queryserver-client';
import { DisposableObject } from './pure/disposable-object';
import { commandRunner } from './commandRunner';
import { assertNever, ONE_HOUR_IN_MS, TWO_HOURS_IN_MS, getErrorMessage, getErrorStack } from './pure/helpers-pure';
import { assertNever, ONE_HOUR_IN_MS, TWO_HOURS_IN_MS, getErrorMessage, getErrorStack } from './pure/helpers-pure';
import { CompletedLocalQueryInfo, LocalQueryInfo as LocalQueryInfo, QueryHistoryInfo } from './query-results';
import { DatabaseManager } from './databases';
import { registerQueryHistoryScubber } from './query-history-scrubber';
import { QueryStatus } from './query-status';
import { slurpQueryHistory, splatQueryHistory } from './query-serialization';
import * as fs from 'fs-extra';
import { CliVersionConstraint } from './cli';
/**
* query-history.ts
@@ -181,38 +183,48 @@ export class HistoryTreeDataProvider extends DisposableObject {
): ProviderResult<QueryHistoryInfo[]> {
return element ? [] : this.history.sort((h1, h2) => {
// TODO remote queries are not implemented yet.
if (h1.t !== 'local' && h2.t !== 'local') {
return 0;
}
if (h1.t !== 'local') {
return -1;
}
if (h2.t !== 'local') {
return 1;
}
const h1Label = h1.label.toLowerCase();
const h2Label = h2.label.toLowerCase();
const resultCount1 = h1.completedQuery?.resultCount ?? -1;
const resultCount2 = h2.completedQuery?.resultCount ?? -1;
const h1Date = h1.t === 'local'
? h1.initialInfo.start.getTime()
: h1.remoteQuery?.executionStartTime;
const h2Date = h2.t === 'local'
? h2.initialInfo.start.getTime()
: h2.remoteQuery?.executionStartTime;
// result count for remote queries is not available here.
const resultCount1 = h1.t === 'local'
? h1.completedQuery?.resultCount ?? -1
: -1;
const resultCount2 = h2.t === 'local'
? h2.completedQuery?.resultCount ?? -1
: -1;
switch (this.sortOrder) {
case SortOrder.NameAsc:
return h1.label.localeCompare(h2.label, env.language);
return h1Label.localeCompare(h2Label, env.language);
case SortOrder.NameDesc:
return h2.label.localeCompare(h1.label, env.language);
return h2Label.localeCompare(h1Label, env.language);
case SortOrder.DateAsc:
return h1.initialInfo.start.getTime() - h2.initialInfo.start.getTime();
return h1Date - h2Date;
case SortOrder.DateDesc:
return h2.initialInfo.start.getTime() - h1.initialInfo.start.getTime();
return h2Date - h1Date;
case SortOrder.CountAsc:
// If the result counts are equal, sort by name.
return resultCount1 - resultCount2 === 0
? h1.label.localeCompare(h2.label, env.language)
? h1Label.localeCompare(h2Label, env.language)
: resultCount1 - resultCount2;
case SortOrder.CountDesc:
// If the result counts are equal, sort by name.
return resultCount2 - resultCount1 === 0
? h2.label.localeCompare(h1.label, env.language)
? h2Label.localeCompare(h1Label, env.language)
: resultCount2 - resultCount1;
default:
assertNever(this.sortOrder);
@@ -406,6 +418,18 @@ export class QueryHistoryManager extends DisposableObject {
this.handleOpenQueryDirectory.bind(this)
)
);
this.push(
commandRunner(
'codeQLQueryHistory.showEvalLog',
this.handleShowEvalLog.bind(this)
)
);
this.push(
commandRunner(
'codeQLQueryHistory.showEvalLogSummary',
this.handleShowEvalLogSummary.bind(this)
)
);
this.push(
commandRunner(
'codeQLQueryHistory.cancel',
@@ -636,7 +660,7 @@ export class QueryHistoryManager extends DisposableObject {
if (response !== undefined) {
// Interpret empty string response as 'go back to using default'
finalSingleItem.initialInfo.userSpecifiedLabel = response === '' ? undefined : response;
this.treeDataProvider.refresh();
await this.refreshTreeView();
}
}
@@ -727,21 +751,72 @@ export class QueryHistoryManager extends DisposableObject {
return;
}
let p: string | undefined;
let externalFilePath: string | undefined;
if (finalSingleItem.t === 'local') {
if (finalSingleItem.completedQuery) {
p = finalSingleItem.completedQuery.query.querySaveDir;
externalFilePath = path.join(finalSingleItem.completedQuery.query.querySaveDir, 'timestamp');
}
} else if (finalSingleItem.t === 'remote') {
p = path.join(this.queryStorageDir, finalSingleItem.queryId);
externalFilePath = path.join(this.queryStorageDir, finalSingleItem.queryId, 'timestamp');
}
if (p) {
try {
await commands.executeCommand('revealFileInOS', Uri.file(p));
} catch (e) {
throw new Error(`Failed to open ${p}: ${getErrorMessage(e)}`);
if (externalFilePath) {
if (!(await fs.pathExists(externalFilePath))) {
// timestamp file is missing (manually deleted?) try selecting the parent folder.
// It's less nice, but at least it will work.
externalFilePath = path.dirname(externalFilePath);
if (!(await fs.pathExists(externalFilePath))) {
throw new Error(`Query directory does not exist: ${externalFilePath}`);
}
}
try {
await commands.executeCommand('revealFileInOS', Uri.file(externalFilePath));
} catch (e) {
throw new Error(`Failed to open ${externalFilePath}: ${getErrorMessage(e)}`);
}
}
}
private warnNoEvalLog() {
void showAndLogWarningMessage('No evaluator log is available for this run. Perhaps it failed before evaluation, or you are running with a version of CodeQL before ' + CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG + '?');
}
async handleShowEvalLog(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[]
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
// Only applicable to an individual local query
if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem || finalSingleItem.t !== 'local') {
return;
}
if (finalSingleItem.evalLogLocation) {
await this.tryOpenExternalFile(finalSingleItem.evalLogLocation);
} else {
this.warnNoEvalLog();
}
}
async handleShowEvalLogSummary(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[]
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
// Only applicable to an individual local query
if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem || finalSingleItem.t !== 'local') {
return;
}
if (finalSingleItem.evalLogLocation) {
if (!fs.existsSync(finalSingleItem.evalLogSummaryLocation)) {
await this.qs.cliServer.generateLogSummary(finalSingleItem.evalLogLocation, finalSingleItem.evalLogSummaryLocation);
}
await this.tryOpenExternalFile(finalSingleItem.evalLogSummaryLocation);
} else {
this.warnNoEvalLog();
}
}

9
extensions/ql-vscode/src/query-results.ts
View File

@@ -216,6 +216,7 @@ export class LocalQueryInfo {
public failureReason: string | undefined;
public completedQuery: CompletedQueryInfo | undefined;
public evalLogLocation: string | undefined;
private config: QueryHistoryConfig | undefined;
/**
@@ -311,6 +312,14 @@ export class LocalQueryInfo {
}
}
/**
* Return the location of a query's evaluator log summary. This file may not exist yet,
* in which case it can be created by invoking `codeql generate log-summary`.
*/
get evalLogSummaryLocation(): string {
return this.evalLogLocation + '.summary';
}
get completed(): boolean {
return !!this.completedQuery;
}

6
extensions/ql-vscode/src/queryserver-client.ts
View File

@@ -146,7 +146,7 @@ export class QueryServerClient extends DisposableObject {
args.push('--require-db-registration');
}
if (await this.cliServer.cliConstraints.supportsOldEvalStats()) {
if (await this.cliServer.cliConstraints.supportsOldEvalStats() && !(await this.cliServer.cliConstraints.supportsPerQueryEvalLog())) {
args.push('--old-eval-stats');
}
@@ -258,3 +258,7 @@ export class QueryServerClient extends DisposableObject {
export function findQueryLogFile(resultPath: string): string {
return path.join(resultPath, 'query.log');
}
export function findQueryStructLogFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.jsonl');
}

23
extensions/ql-vscode/src/run-queries.ts
View File

@@ -29,7 +29,7 @@ import { ProgressCallback, UserCancellationException } from './commandRunner';
import { DatabaseInfo, QueryMetadata } from './pure/interface-types';
import { logger } from './logging';
import * as messages from './pure/messages';
import { InitialQueryInfo } from './query-results';
import { InitialQueryInfo, LocalQueryInfo } from './query-results';
import * as qsClient from './queryserver-client';
import { isQuickQueryPath } from './quick-query';
import { compileDatabaseUpgradeSequence, hasNondestructiveUpgradeCapabilities, upgradeDatabaseExplicit } from './upgrades';
@@ -95,6 +95,10 @@ export class QueryEvaluationInfo {
return qsClient.findQueryLogFile(this.querySaveDir);
}
get structLogPath() {
return qsClient.findQueryStructLogFile(this.querySaveDir);
}
get resultsPaths() {
return {
resultsPath: path.join(this.querySaveDir, 'results.bqrs'),
@@ -125,6 +129,7 @@ export class QueryEvaluationInfo {
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
queryInfo?: LocalQueryInfo,
): Promise<messages.EvaluationResult> {
if (!dbItem.contents || dbItem.error) {
throw new Error('Can\'t run query on invalid database.');
@@ -156,6 +161,12 @@ export class QueryEvaluationInfo {
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default'
};
if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
await qs.sendRequest(messages.startLog, {
db: dataset,
logPath: this.structLogPath,
});
}
const params: messages.EvaluateQueriesParams = {
db: dataset,
evaluateId: callbackId,
@@ -172,6 +183,13 @@ export class QueryEvaluationInfo {
}
} finally {
qs.unRegisterCallback(callbackId);
if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
await qs.sendRequest(messages.endLog, {
db: dataset,
logPath: this.structLogPath,
});
queryInfo.evalLogLocation = this.structLogPath;
}
}
return result || {
evaluationTime: 0,
@@ -658,6 +676,7 @@ export async function compileAndRunQueryAgainstDatabase(
progress: ProgressCallback,
token: CancellationToken,
templates?: messages.TemplateDefinitions,
queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
): Promise<QueryWithResults> {
if (!dbItem.contents || !dbItem.contents.dbSchemeUri) {
throw new Error(`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`);
@@ -743,7 +762,7 @@ export async function compileAndRunQueryAgainstDatabase(
}
if (errors.length === 0) {
const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token);
const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token, queryInfo);
if (result.resultType !== messages.QueryResultType.SUCCESS) {
const message = result.message || 'Failed to run query';
void logger.log(message);

27
extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts
View File

@@ -12,6 +12,7 @@ import {
convertLgtmUrlToDatabaseUrl,
looksLikeLgtmUrl,
findDirWithFile,
looksLikeGithubRepo,
} from '../../databaseFetcher';
import { ProgressCallback } from '../../commandRunner';
import * as pq from 'proxyquire';
@@ -209,6 +210,32 @@ describe('databaseFetcher', function() {
});
});
describe('looksLikeGithubRepo', () => {
it('should handle invalid urls', () => {
expect(looksLikeGithubRepo(''))
.to.be.false;
expect(looksLikeGithubRepo('http://github.com/foo/bar'))
.to.be.false;
expect(looksLikeGithubRepo('https://ww.github.com/foo/bar'))
.to.be.false;
expect(looksLikeGithubRepo('https://ww.github.com/foo'))
.to.be.false;
expect(looksLikeGithubRepo('foo'))
.to.be.false;
});
it('should handle valid urls', () => {
expect(looksLikeGithubRepo('https://github.com/foo/bar'))
.to.be.true;
expect(looksLikeGithubRepo('https://www.github.com/foo/bar'))
.to.be.true;
expect(looksLikeGithubRepo('https://github.com/foo/bar/sub/pages'))
.to.be.true;
expect(looksLikeGithubRepo('foo/bar'))
.to.be.true;
});
});
describe('looksLikeLgtmUrl', () => {
it('should handle invalid urls', () => {
expect(looksLikeLgtmUrl('')).to.be.false;

48
extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts
View File

@@ -427,9 +427,11 @@ describe('query-history', () => {
describe('getChildren', () => {
const history = [
item('a', 10, 20),
item('b', 5, 30),
item('c', 1, 25),
item('a', 2, 'remote'),
item('b', 10, 'local', 20),
item('c', 5, 'local', 30),
item('d', 1, 'local', 25),
item('e', 6, 'remote'),
];
let treeDataProvider: HistoryTreeDataProvider;
@@ -456,7 +458,7 @@ describe('query-history', () => {
});
it('should get children for date ascending', async () => {
const expected = [history[2], history[1], history[0]];
const expected = [history[3], history[0], history[2], history[4], history[1]];
treeDataProvider.sortOrder = SortOrder.DateAsc;
const children = await treeDataProvider.getChildren();
@@ -464,7 +466,7 @@ describe('query-history', () => {
});
it('should get children for date descending', async () => {
const expected = [history[0], history[1], history[2]];
const expected = [history[3], history[0], history[2], history[4], history[1]].reverse();
treeDataProvider.sortOrder = SortOrder.DateDesc;
const children = await treeDataProvider.getChildren();
@@ -472,7 +474,7 @@ describe('query-history', () => {
});
it('should get children for result count ascending', async () => {
const expected = [history[0], history[2], history[1]];
const expected = [history[0], history[4], history[1], history[3], history[2]];
treeDataProvider.sortOrder = SortOrder.CountAsc;
const children = await treeDataProvider.getChildren();
@@ -480,7 +482,7 @@ describe('query-history', () => {
});
it('should get children for result count descending', async () => {
const expected = [history[1], history[2], history[0]];
const expected = [history[0], history[4], history[1], history[3], history[2]].reverse();
treeDataProvider.sortOrder = SortOrder.CountDesc;
const children = await treeDataProvider.getChildren();
@@ -509,17 +511,27 @@ describe('query-history', () => {
expect(children).to.deep.eq(expected);
});
function item(label: string, start: number, resultCount?: number) {
return {
label,
initialInfo: {
start: new Date(start),
},
completedQuery: {
resultCount,
},
t: 'local'
};
function item(label: string, start: number, t = 'local', resultCount?: number) {
if (t === 'local') {
return {
label,
initialInfo: {
start: new Date(start),
},
completedQuery: {
resultCount,
},
t
};
} else {
return {
label,
remoteQuery: {
executionStartTime: start,
},
t
};
}
}
});

6
extensions/ql-vscode/src/vscode-tests/no-workspace/remote-query-history.test.ts
View File

@@ -337,13 +337,13 @@ describe('Remote queries and query history manager', function() {
// Load remoteQueryResult0.analysisSummaries[1] into memory
await arm.downloadAnalysisResults(remoteQueryResult0.analysisSummaries[1], () => Promise.resolve());
expect(await (arm as any).isDownloadedNotInMemory(remoteQueryResult0.analysisSummaries[0])).to.be.true;
expect(await (arm as any).isAnalysisDownloadedNotInMemory(remoteQueryResult0.analysisSummaries[0])).to.be.true;
// in memory
expect(await (arm as any).isDownloadedNotInMemory(remoteQueryResult0.analysisSummaries[1])).to.be.false;
expect(await (arm as any).isAnalysisDownloadedNotInMemory(remoteQueryResult0.analysisSummaries[1])).to.be.false;
// not downloaded
expect(await (arm as any).isDownloadedNotInMemory(remoteQueryResult0.analysisSummaries[2])).to.be.false;
expect(await (arm as any).isAnalysisDownloadedNotInMemory(remoteQueryResult0.analysisSummaries[2])).to.be.false;
});
it('should load downloaded artifacts', async () => {