Merge branch 'main' into starcke/commands-registration

This commit is contained in:
Koen Vlaswinkel
2023-03-14 12:55:25 +01:00
committed by GitHub
45 changed files with 851 additions and 421 deletions

View File

@@ -2,6 +2,8 @@ name: "CodeQL config"
queries:
- name: Run standard queries
uses: security-and-quality
- name: Experimental queries
uses: security-experimental
- name: Run custom javascript queries
uses: ./.github/codeql/queries
paths:

15
.vscode/settings.json vendored
View File

@@ -42,22 +42,29 @@
"LANG": "en-US",
"TZ": "UTC"
},
// These options are used by the `jestrunner.debug` command.
// They are not used by the `jestrunner.run` command.
// After clicking "debug" over a test, continually invoke the
// "Debug: Attach to Node Process" command until you see a
// process named "Code Helper (Plugin)". Then click "attach".
// This will attach the debugger to the test process.
"jestrunner.debugOptions": {
// Uncomment to debug integration tests
// "attachSimplePort": 9223,
"attachSimplePort": 9223,
"env": {
"LANG": "en-US",
"TZ": "UTC",
// Uncomment to set a custom path to a CodeQL checkout.
// "TEST_CODEQL_PATH": "../codeql",
// "TEST_CODEQL_PATH": "/absolute/path/to/checkout/of/codeql",
// Uncomment to set a custom path to a CodeQL CLI executable.
// This is the CodeQL version that will be used in the tests.
// "CLI_PATH": "/path/to/customg/codeql",
// "CLI_PATH": "/absolute/path/to/custom/codeql",
// Uncomment to debug integration tests
// "VSCODE_WAIT_FOR_DEBUGGER": "true",
"VSCODE_WAIT_FOR_DEBUGGER": "true",
}
},
"terminal.integrated.env.linux": {

View File

@@ -23,9 +23,9 @@
* Wait for the PR to be merged into `main`
1. Switch to `main` branch and pull latest changes
1. Lock the `main` branch.
* Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
* Select "Lock branch"
* Click "Save changes"
* Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
* Select "Lock branch"
* Click "Save changes"
1. Ensure that no PRs have been merged since the release PR that you merged. If there were, you might need to unlock `main` temporarily and update the CHANGELOG again.
1. Build the extension `npm run build` and install it on your VS Code using "Install from VSIX".
1. Go through [our test plan](./test-plan.md) to ensure that the extension is working as expected.
@@ -40,9 +40,9 @@
git tag -d badly-named-tag
```
1. Unlock the main branch
* Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
* Deselect "Lock branch"
* Click "Save changes"
* Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
* Deselect "Lock branch"
* Click "Save changes"
1. Push the new tag up:
a. If you're using a fork of the repo:
@@ -86,4 +86,4 @@ To regenerate the Open VSX token:
1. Go to the [Access Tokens](https://open-vsx.org/user-settings/tokens) page and generate a new token.
1. Update the secret in the `publish-open-vsx` environment in the project settings.
To regenerate the VSCode Marketplace token, please see our internal documentation. Note that Azure DevOps PATs expire every 90 days and must be regenerated.
To regenerate the VSCode Marketplace token, please see our internal documentation. Note that Azure DevOps PATs expire every 90 days and must be regenerated.

View File

@@ -1,6 +1,8 @@
# CodeQL for Visual Studio Code: Changelog
## 1.8.0 - 8 March 2023
## [UNRELEASED]
## 1.8.0 - 9 March 2023
- Send telemetry about unhandled errors happening within the extension. [#2125](https://github.com/github/vscode-codeql/pull/2125)
- Enable multi-repository variant analysis. [#2144](https://github.com/github/vscode-codeql/pull/2144)

View File

@@ -93,12 +93,6 @@ export async function deployPackage(
);
await copyPackage(sourcePath, distPath);
// This is necessary for vsce to know the dependencies
await copyDirectory(
resolve(sourcePath, "node_modules"),
resolve(distPath, "node_modules"),
);
return {
distPath,
name: packageJson.name,

View File

@@ -17,6 +17,7 @@ export async function packageExtension(): Promise<void> {
"..",
`${deployedPackage.name}-${deployedPackage.version}.vsix`,
),
"--no-dependencies",
];
const proc = spawn(resolve(__dirname, "../node_modules/.bin/vsce"), args, {
cwd: deployedPackage.distPath,

View File

@@ -1,12 +1,12 @@
{
"name": "vscode-codeql",
"version": "1.8.0",
"version": "1.8.1",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "vscode-codeql",
"version": "1.8.0",
"version": "1.8.1",
"hasInstallScript": true,
"license": "MIT",
"dependencies": {
@@ -105,7 +105,6 @@
"@vscode/vsce": "^2.15.0",
"ansi-colors": "^4.1.1",
"applicationinsights": "^2.3.5",
"babel-loader": "^8.2.5",
"cross-env": "^7.0.3",
"css-loader": "~3.1.0",
"del": "^6.0.0",

View File

@@ -4,7 +4,7 @@
"description": "CodeQL for Visual Studio Code",
"author": "GitHub",
"private": true,
"version": "1.8.0",
"version": "1.8.1",
"publisher": "GitHub",
"license": "MIT",
"icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -44,11 +44,6 @@
"onView:test-explorer",
"onCommand:codeQL.checkForUpdatesToCLI",
"onCommand:codeQL.authenticateToGitHub",
"onCommand:codeQLDatabases.chooseDatabaseFolder",
"onCommand:codeQLDatabases.chooseDatabaseArchive",
"onCommand:codeQLDatabases.chooseDatabaseInternet",
"onCommand:codeQLDatabases.chooseDatabaseGithub",
"onCommand:codeQL.setCurrentDatabase",
"onCommand:codeQL.viewAst",
"onCommand:codeQL.viewCfg",
"onCommand:codeQL.openReferencedFile",
@@ -57,16 +52,6 @@
"onCommand:codeQL.chooseDatabaseArchive",
"onCommand:codeQL.chooseDatabaseInternet",
"onCommand:codeQL.chooseDatabaseGithub",
"onCommand:codeQLDatabases.chooseDatabase",
"onCommand:codeQLDatabases.setCurrentDatabase",
"onCommand:codeQLVariantAnalysisRepositories.openConfigFile",
"onCommand:codeQLVariantAnalysisRepositories.addNewDatabase",
"onCommand:codeQLVariantAnalysisRepositories.addNewList",
"onCommand:codeQLVariantAnalysisRepositories.setSelectedItem",
"onCommand:codeQLVariantAnalysisRepositories.setSelectedItemContextMenu",
"onCommand:codeQLVariantAnalysisRepositories.renameItemContextMenu",
"onCommand:codeQLVariantAnalysisRepositories.openOnGitHubContextMenu",
"onCommand:codeQLVariantAnalysisRepositories.removeItemContextMenu",
"onCommand:codeQL.quickQuery",
"onCommand:codeQL.restartQueryServer",
"onWebviewPanel:resultsView",
@@ -239,6 +224,19 @@
"default": true,
"description": "Enable the 'Quick Evaluation' CodeLens."
},
"codeQL.runningQueries.useExtensionPacks": {
"type": "string",
"default": "none",
"enum": [
"none",
"all"
],
"enumDescriptions": [
"Do not use extension packs.",
"Use all extension packs found in the workspace."
],
"description": "Choose whether or not to run queries using extension packs. Requires CodeQL CLI v2.12.3 or later."
},
"codeQL.resultsDisplay.pageSize": {
"type": "integer",
"default": 200,
@@ -322,6 +320,10 @@
"command": "codeQL.runVariantAnalysis",
"title": "CodeQL: Run Variant Analysis"
},
{
"command": "codeQL.runVariantAnalysisContextEditor",
"title": "CodeQL: Run Variant Analysis"
},
{
"command": "codeQL.exportSelectedVariantAnalysisResults",
"title": "CodeQL: Export Variant Analysis Results"
@@ -334,10 +336,22 @@
"command": "codeQL.quickEval",
"title": "CodeQL: Quick Evaluation"
},
{
"command": "codeQL.quickEvalContextEditor",
"title": "CodeQL: Quick Evaluation"
},
{
"command": "codeQL.openReferencedFile",
"title": "CodeQL: Open Referenced File"
},
{
"command": "codeQL.openReferencedFileContextEditor",
"title": "CodeQL: Open Referenced File"
},
{
"command": "codeQL.openReferencedFileContextExplorer",
"title": "CodeQL: Open Referenced File"
},
{
"command": "codeQL.previewQueryHelp",
"title": "CodeQL: Preview Query Help"
@@ -433,10 +447,26 @@
"command": "codeQL.viewAst",
"title": "CodeQL: View AST"
},
{
"command": "codeQL.viewAstContextExplorer",
"title": "CodeQL: View AST"
},
{
"command": "codeQL.viewAstContextEditor",
"title": "CodeQL: View AST"
},
{
"command": "codeQL.viewCfg",
"title": "CodeQL: View CFG"
},
{
"command": "codeQL.viewCfgContextExplorer",
"title": "CodeQL: View CFG"
},
{
"command": "codeQL.viewCfgContextEditor",
"title": "CodeQL: View CFG"
},
{
"command": "codeQL.upgradeCurrentDatabase",
"title": "CodeQL: Upgrade Current Database"
@@ -825,12 +855,12 @@
{
"command": "codeQLQueryHistory.removeHistoryItem",
"group": "7_queryHistory@0",
"when": "viewItem == interpretedResultsItem || viewItem == rawResultsItem || viewItem == remoteResultsItem || viewItem == cancelledResultsItem || viewItem == cancelledRemoteResultsItem"
"when": "viewItem == interpretedResultsItem || viewItem == rawResultsItem || viewItem == remoteResultsItem || viewItem == cancelledRemoteResultsItemWithoutLogs || viewItem == cancelledResultsItem || viewItem == cancelledRemoteResultsItem"
},
{
"command": "codeQLQueryHistory.removeHistoryItem",
"group": "inline",
"when": "viewItem == interpretedResultsItem || viewItem == rawResultsItem || viewItem == remoteResultsItem || viewItem == cancelledResultsItem || viewItem == cancelledRemoteResultsItem"
"when": "viewItem == interpretedResultsItem || viewItem == rawResultsItem || viewItem == remoteResultsItem || viewItem == cancelledRemoteResultsItemWithoutLogs || viewItem == cancelledResultsItem || viewItem == cancelledRemoteResultsItem"
},
{
"command": "codeQLQueryHistory.renameItem",
@@ -930,12 +960,12 @@
"when": "resourceScheme == codeql-zip-archive || explorerResourceIsFolder || resourceExtname == .zip"
},
{
"command": "codeQL.viewAst",
"command": "codeQL.viewAstContextExplorer",
"group": "9_qlCommands",
"when": "resourceScheme == codeql-zip-archive && !explorerResourceIsFolder && !listMultiSelection"
},
{
"command": "codeQL.viewCfg",
"command": "codeQL.viewCfgContextExplorer",
"group": "9_qlCommands",
"when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
},
@@ -945,7 +975,7 @@
"when": "resourceScheme != codeql-zip-archive"
},
{
"command": "codeQL.openReferencedFile",
"command": "codeQL.openReferencedFileContextExplorer",
"group": "9_qlCommands",
"when": "resourceExtname == .qlref"
},
@@ -981,7 +1011,8 @@
"when": "editorLangId == ql && resourceExtname == .ql"
},
{
"command": "codeQL.exportSelectedVariantAnalysisResults"
"command": "codeQL.runVariantAnalysisContextEditor",
"when": "false"
},
{
"command": "codeQL.runQueries",
@@ -991,10 +1022,22 @@
"command": "codeQL.quickEval",
"when": "editorLangId == ql"
},
{
"command": "codeQL.quickEvalContextEditor",
"when": "false"
},
{
"command": "codeQL.openReferencedFile",
"when": "resourceExtname == .qlref"
},
{
"command": "codeQL.openReferencedFileContextEditor",
"when": "false"
},
{
"command": "codeQL.openReferencedFileContextExplorer",
"when": "false"
},
{
"command": "codeQL.previewQueryHelp",
"when": "resourceExtname == .qhelp && isWorkspaceTrusted"
@@ -1007,10 +1050,26 @@
"command": "codeQL.viewAst",
"when": "resourceScheme == codeql-zip-archive"
},
{
"command": "codeQL.viewAstContextEditor",
"when": "false"
},
{
"command": "codeQL.viewAstContextExplorer",
"when": "false"
},
{
"command": "codeQL.viewCfg",
"when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
},
{
"command": "codeQL.viewCfgContextExplorer",
"when": "false"
},
{
"command": "codeQL.viewCfgContextEditor",
"when": "false"
},
{
"command": "codeQLVariantAnalysisRepositories.openConfigFile",
"when": "false"
@@ -1234,23 +1293,23 @@
"when": "editorLangId == ql && resourceExtname == .ql"
},
{
"command": "codeQL.runVariantAnalysis",
"command": "codeQL.runVariantAnalysisContextEditor",
"when": "editorLangId == ql && resourceExtname == .ql"
},
{
"command": "codeQL.viewAst",
"command": "codeQL.viewAstContextEditor",
"when": "resourceScheme == codeql-zip-archive"
},
{
"command": "codeQL.viewCfg",
"command": "codeQL.viewCfgContextEditor",
"when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
},
{
"command": "codeQL.quickEval",
"command": "codeQL.quickEvalContextEditor",
"when": "editorLangId == ql"
},
{
"command": "codeQL.openReferencedFile",
"command": "codeQL.openReferencedFileContextEditor",
"when": "resourceExtname == .qlref"
},
{
@@ -1439,7 +1498,6 @@
"@vscode/vsce": "^2.15.0",
"ansi-colors": "^4.1.1",
"applicationinsights": "^2.3.5",
"babel-loader": "^8.2.5",
"cross-env": "^7.0.3",
"css-loader": "~3.1.0",
"del": "^6.0.0",

View File

@@ -1163,24 +1163,32 @@ export class CodeQLCliServer implements Disposable {
/**
* Gets information about available qlpacks
* @param additionalPacks A list of directories to search for qlpacks before searching in `searchPath`.
* @param searchPath A list of directories to search for packs not found in `additionalPacks`. If undefined,
* the default CLI search path is used.
* @param additionalPacks A list of directories to search for qlpacks.
* @param extensionPacksOnly Whether to only search for extension packs. If true, only extension packs will
* be returned. If false, all packs will be returned.
* @returns A dictionary mapping qlpack name to the directory it comes from
*/
resolveQlpacks(
async resolveQlpacks(
additionalPacks: string[],
searchPath?: string[],
extensionPacksOnly = false,
): Promise<QlpacksInfo> {
const args = this.getAdditionalPacksArg(additionalPacks);
if (searchPath?.length) {
args.push("--search-path", join(...searchPath));
if (extensionPacksOnly) {
if (!(await this.cliConstraints.supportsQlpacksKind())) {
void this.logger.log(
"Warning: Running with extension packs is only supported by CodeQL CLI v2.12.3 or later.",
);
return {};
}
args.push("--kind", "extension", "--no-recursive");
}
return this.runJsonCodeQlCliCommand<QlpacksInfo>(
["resolve", "qlpacks"],
args,
"Resolving qlpack information",
`Resolving qlpack information${
extensionPacksOnly ? " (extension packs only)" : ""
}`,
);
}
@@ -1380,6 +1388,17 @@ export class CodeQLCliServer implements Disposable {
private getAdditionalPacksArg(paths: string[]): string[] {
return paths.length ? ["--additional-packs", paths.join(delimiter)] : [];
}
public async useExtensionPacks(): Promise<boolean> {
return (
this.cliConfig.useExtensionPacks &&
(await this.cliConstraints.supportsQlpacksKind())
);
}
public async setUseExtensionPacks(useExtensionPacks: boolean) {
await this.cliConfig.setUseExtensionPacks(useExtensionPacks);
}
}
/**
@@ -1668,6 +1687,11 @@ export class CliVersionConstraint {
*/
public static CLI_VERSION_WITH_WORKSPACE_RFERENCES = new SemVer("2.11.3");
/**
* CLI version that supports the `--kind` option for the `resolve qlpacks` command.
*/
public static CLI_VERSION_WITH_QLPACKS_KIND = new SemVer("2.12.3");
constructor(private readonly cli: CodeQLCliServer) {
/**/
}
@@ -1725,4 +1749,10 @@ export class CliVersionConstraint {
CliVersionConstraint.CLI_VERSION_WITH_WORKSPACE_RFERENCES,
);
}
async supportsQlpacksKind() {
return this.isVersionAtLeast(
CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND,
);
}
}

View File

@@ -1,6 +1,6 @@
import {
CancellationToken,
ProgressOptions,
ProgressOptions as VSCodeProgressOptions,
window as Window,
commands,
Disposable,
@@ -42,22 +42,40 @@ export interface ProgressUpdate {
export type ProgressCallback = (p: ProgressUpdate) => void;
// Make certain properties within a type optional
type Optional<T, K extends keyof T> = Pick<Partial<T>, K> & Omit<T, K>;
export type ProgressOptions = Optional<VSCodeProgressOptions, "location">;
/**
* A task that reports progress.
*
* @param progress a progress handler function. Call this
* function with a `ProgressUpdate` instance in order to
* denote some progress being achieved on this task.
* @param token a cancellation token
*/
export type ProgressTask<R> = (
progress: ProgressCallback,
token: CancellationToken,
) => Thenable<R>;
/**
* A task that handles command invocations from `commandRunner`
* and includes a progress monitor.
*
*
* Arguments passed to the command handler are passed along,
* untouched to this `ProgressTask` instance.
* untouched to this `ProgressTaskWithArgs` instance.
*
* @param progress a progress handler function. Call this
* function with a `ProgressUpdate` instance in order to
* denote some progress being achieved on this task.
* @param token a cencellation token
* @param token a cancellation token
* @param args arguments passed to this task passed on from
* `commands.registerCommand`.
*/
export type ProgressTask<R> = (
export type ProgressTaskWithArgs<R> = (
progress: ProgressCallback,
token: CancellationToken,
...args: any[]
@@ -90,23 +108,29 @@ type NoProgressTask = (...args: any[]) => Promise<any>;
* request).
*/
export function withProgress<R>(
options: ProgressOptions,
task: ProgressTask<R>,
...args: any[]
{
location = ProgressLocation.Notification,
title,
cancellable,
}: ProgressOptions = {},
): Thenable<R> {
let progressAchieved = 0;
return Window.withProgress(options, (progress, token) => {
return task(
(p) => {
return Window.withProgress(
{
location,
title,
cancellable,
},
(progress, token) => {
return task((p) => {
const { message, step, maxStep } = p;
const increment = (100 * (step - progressAchieved)) / maxStep;
progressAchieved = step;
progress.report({ message, increment });
},
token,
...args,
);
});
}, token);
},
);
}
/**
@@ -121,6 +145,7 @@ export function withProgress<R>(
export function commandRunner(
commandId: string,
task: NoProgressTask,
outputLogger = extLogger,
): Disposable {
return commands.registerCommand(commandId, async (...args: any[]) => {
const startTime = Date.now();
@@ -134,64 +159,6 @@ export function commandRunner(
getErrorMessage(e) || e
} (${commandId})`;
const errorStack = getErrorStack(e);
if (e instanceof UserCancellationException) {
// User has cancelled this action manually
if (e.silent) {
void extLogger.log(errorMessage.fullMessage);
} else {
void showAndLogWarningMessage(errorMessage.fullMessage);
}
} else {
// Include the full stack in the error log only.
const fullMessage = errorStack
? `${errorMessage.fullMessage}\n${errorStack}`
: errorMessage.fullMessage;
void showAndLogExceptionWithTelemetry(errorMessage, {
fullMessage,
extraTelemetryProperties: {
command: commandId,
},
});
}
return undefined;
} finally {
const executionTime = Date.now() - startTime;
telemetryListener?.sendCommandUsage(commandId, executionTime, error);
}
});
}
/**
* A generic wrapper for command registration. This wrapper adds uniform error handling,
* progress monitoring, and cancellation for commands.
*
* @param commandId The ID of the command to register.
* @param task The task to run. It is passed directly to `commands.registerCommand`. Any
* arguments to the command handler are passed on to the task after the progress callback
* and cancellation token.
* @param progressOptions Progress options to be sent to the progress monitor.
*/
export function commandRunnerWithProgress<R>(
commandId: string,
task: ProgressTask<R>,
progressOptions: Partial<ProgressOptions>,
outputLogger = extLogger,
): Disposable {
return commands.registerCommand(commandId, async (...args: any[]) => {
const startTime = Date.now();
let error: Error | undefined;
const progressOptionsWithDefaults = {
location: ProgressLocation.Notification,
...progressOptions,
};
try {
return await withProgress(progressOptionsWithDefaults, task, ...args);
} catch (e) {
error = asError(e);
const errorMessage = redactableError`${
getErrorMessage(e) || e
} (${commandId})`;
const errorStack = getErrorStack(e);
if (e instanceof UserCancellationException) {
// User has cancelled this action manually
if (e.silent) {
@@ -222,6 +189,34 @@ export function commandRunnerWithProgress<R>(
});
}
/**
* A generic wrapper for command registration. This wrapper adds uniform error handling,
* progress monitoring, and cancellation for commands.
*
* @param commandId The ID of the command to register.
* @param task The task to run. It is passed directly to `commands.registerCommand`. Any
* arguments to the command handler are passed on to the task after the progress callback
* and cancellation token.
* @param progressOptions Progress options to be sent to the progress monitor.
*/
export function commandRunnerWithProgress<R>(
commandId: string,
task: ProgressTaskWithArgs<R>,
progressOptions: ProgressOptions,
outputLogger = extLogger,
): Disposable {
return commandRunner(
commandId,
async (...args: any[]) => {
return withProgress(
(progress, token) => task(progress, token, ...args),
progressOptions,
);
},
outputLogger,
);
}
/**
* Displays a progress monitor that indicates how much progess has been made
* reading from a stream.

View File

@@ -1,3 +1,4 @@
export * from "./logger";
export * from "./tee-logger";
export * from "./vscode/loggers";
export * from "./vscode/output-channel-logger";

View File

@@ -1,9 +1,6 @@
export interface LogOptions {
// If false, don't output a trailing newline for the log entry. Default true.
trailingNewline?: boolean;
// If specified, add this log entry to the log file at the specified location.
additionalLogLocation?: string;
}
export interface Logger {
@@ -25,11 +22,4 @@ export interface Logger {
* @param preserveFocus When `true` the channel will not take focus.
*/
show(preserveFocus?: boolean): void;
/**
* Remove the log at the specified location.
*
* @param location log to remove
*/
removeAdditionalLogLocation(location: string | undefined): void;
}

View File

@@ -0,0 +1,68 @@
import { appendFile, ensureFile } from "fs-extra";
import { isAbsolute } from "path";
import { getErrorMessage } from "../../pure/helpers-pure";
import { Logger, LogOptions } from "./logger";
/**
* An implementation of {@link Logger} that sends the output both to another {@link Logger}
* and to a file.
*
* The first time a message is written, an additional banner is written to the underlying logger
* pointing the user to the "side log" file.
*/
export class TeeLogger implements Logger {
private emittedRedirectMessage = false;
private error = false;
public constructor(
private readonly logger: Logger,
private readonly location: string,
) {
if (!isAbsolute(location)) {
throw new Error(
`Additional Log Location must be an absolute path: ${location}`,
);
}
}
async log(message: string, options = {} as LogOptions): Promise<void> {
if (!this.emittedRedirectMessage) {
this.emittedRedirectMessage = true;
const msg = `| Log being saved to ${this.location} |`;
const separator = new Array(msg.length).fill("-").join("");
await this.logger.log(separator);
await this.logger.log(msg);
await this.logger.log(separator);
}
if (!this.error) {
try {
const trailingNewline = options.trailingNewline ?? true;
await ensureFile(this.location);
await appendFile(
this.location,
message + (trailingNewline ? "\n" : ""),
{
encoding: "utf8",
},
);
} catch (e) {
// Write an error message to the primary log, and stop trying to write to the side log.
this.error = true;
const errorMessage = getErrorMessage(e);
await this.logger.log(
`Error writing to additional log file: ${errorMessage}`,
);
}
}
if (!this.error) {
await this.logger.log(message, options);
}
}
show(preserveFocus?: boolean): void {
this.logger.show(preserveFocus);
}
}

View File

@@ -1,6 +1,4 @@
import { window as Window, OutputChannel, Progress } from "vscode";
import { ensureFile, appendFile } from "fs-extra";
import { isAbsolute } from "path";
import { Logger, LogOptions } from "../logger";
import { DisposableObject } from "../../../pure/disposable-object";
@@ -9,10 +7,6 @@ import { DisposableObject } from "../../../pure/disposable-object";
*/
export class OutputChannelLogger extends DisposableObject implements Logger {
public readonly outputChannel: OutputChannel;
private readonly additionalLocations = new Map<
string,
AdditionalLogLocation
>();
isCustomLogDirectory: boolean;
constructor(title: string) {
@@ -32,27 +26,6 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
} else {
this.outputChannel.append(message);
}
if (options.additionalLogLocation) {
if (!isAbsolute(options.additionalLogLocation)) {
throw new Error(
`Additional Log Location must be an absolute path: ${options.additionalLogLocation}`,
);
}
const logPath = options.additionalLogLocation;
let additional = this.additionalLocations.get(logPath);
if (!additional) {
const msg = `| Log being saved to ${logPath} |`;
const separator = new Array(msg.length).fill("-").join("");
this.outputChannel.appendLine(separator);
this.outputChannel.appendLine(msg);
this.outputChannel.appendLine(separator);
additional = new AdditionalLogLocation(logPath);
this.additionalLocations.set(logPath, additional);
}
await additional.log(message, options);
}
} catch (e) {
if (e instanceof Error && e.message === "Channel has been closed") {
// Output channel is closed logging to console instead
@@ -69,31 +42,6 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
show(preserveFocus?: boolean): void {
this.outputChannel.show(preserveFocus);
}
removeAdditionalLogLocation(location: string | undefined): void {
if (location) {
this.additionalLocations.delete(location);
}
}
}
class AdditionalLogLocation {
constructor(private location: string) {}
async log(message: string, options = {} as LogOptions): Promise<void> {
if (options.trailingNewline === undefined) {
options.trailingNewline = true;
}
await ensureFile(this.location);
await appendFile(
this.location,
message + (options.trailingNewline ? "\n" : ""),
{
encoding: "utf8",
},
);
}
}
export type ProgressReporter = Progress<{ message: string }>;

View File

@@ -137,6 +137,10 @@ const DEBUG_SETTING = new Setting("debug", RUNNING_QUERIES_SETTING);
const MAX_PATHS = new Setting("maxPaths", RUNNING_QUERIES_SETTING);
const RUNNING_TESTS_SETTING = new Setting("runningTests", ROOT_SETTING);
const RESULTS_DISPLAY_SETTING = new Setting("resultsDisplay", ROOT_SETTING);
const USE_EXTENSION_PACKS = new Setting(
"useExtensionPacks",
RUNNING_QUERIES_SETTING,
);
export const ADDITIONAL_TEST_ARGUMENTS_SETTING = new Setting(
"additionalTestArguments",
@@ -196,6 +200,7 @@ const CLI_SETTINGS = [
NUMBER_OF_TEST_THREADS_SETTING,
NUMBER_OF_THREADS_SETTING,
MAX_PATHS,
USE_EXTENSION_PACKS,
];
export interface CliConfig {
@@ -203,7 +208,9 @@ export interface CliConfig {
numberTestThreads: number;
numberThreads: number;
maxPaths: number;
useExtensionPacks: boolean;
onDidChangeConfiguration?: Event<void>;
setUseExtensionPacks: (useExtensionPacks: boolean) => Promise<void>;
}
export abstract class ConfigListener extends DisposableObject {
@@ -400,6 +407,19 @@ export class CliConfigListener extends ConfigListener implements CliConfig {
return MAX_PATHS.getValue<number>();
}
public get useExtensionPacks(): boolean {
// currently, we are restricting the values of this setting to 'all' or 'none'.
return USE_EXTENSION_PACKS.getValue() === "all";
}
// Exposed for testing only
public async setUseExtensionPacks(newUseExtensionPacks: boolean) {
await USE_EXTENSION_PACKS.updateValue(
newUseExtensionPacks ? "all" : "none",
ConfigurationTarget.Global,
);
}
protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
this.handleDidChangeConfigurationForRelevantSettings(CLI_SETTINGS, e);
}

View File

@@ -4,7 +4,6 @@ import {
Location,
LocationLink,
Position,
ProgressLocation,
ReferenceContext,
ReferenceProvider,
TextDocument,
@@ -73,11 +72,6 @@ export class TemplateQueryDefinitionProvider implements DefinitionProvider {
private async getDefinitions(uriString: string): Promise<LocationLink[]> {
return withProgress(
{
location: ProgressLocation.Notification,
cancellable: true,
title: "Finding definitions",
},
async (progress, token) => {
return getLocationsForUriString(
this.cli,
@@ -91,6 +85,10 @@ export class TemplateQueryDefinitionProvider implements DefinitionProvider {
(src, _dest) => src === uriString,
);
},
{
cancellable: true,
title: "Finding definitions",
},
);
}
}
@@ -136,11 +134,6 @@ export class TemplateQueryReferenceProvider implements ReferenceProvider {
private async getReferences(uriString: string): Promise<FullLocationLink[]> {
return withProgress(
{
location: ProgressLocation.Notification,
cancellable: true,
title: "Finding references",
},
async (progress, token) => {
return getLocationsForUriString(
this.cli,
@@ -154,6 +147,10 @@ export class TemplateQueryReferenceProvider implements ReferenceProvider {
(src, _dest) => src === uriString,
);
},
{
cancellable: true,
title: "Finding references",
},
);
}
}

View File

@@ -9,7 +9,6 @@ import {
extensions,
languages,
ProgressLocation,
ProgressOptions,
QuickPickItem,
Range,
Uri,
@@ -331,16 +330,15 @@ export async function activate(
await commands.executeCommand("workbench.action.reloadWindow");
}
} else {
const progressOptions: ProgressOptions = {
title: progressTitle,
location: ProgressLocation.Notification,
};
await withProgress(progressOptions, (progress) =>
distributionManager.installExtensionManagedDistributionRelease(
result.updatedRelease,
progress,
),
await withProgress(
(progress) =>
distributionManager.installExtensionManagedDistributionRelease(
result.updatedRelease,
progress,
),
{
title: progressTitle,
},
);
await ctx.globalState.update(shouldUpdateOnNextActivationKey, false);
@@ -1083,6 +1081,7 @@ async function activateWithInstalledDistribution(
queryServerLogger,
),
);
ctx.subscriptions.push(
commandRunnerWithProgress(
"codeQL.quickEval",
@@ -1100,6 +1099,24 @@ async function activateWithInstalledDistribution(
),
);
// Since we are tracking extension usage through commands, this command mirrors the "codeQL.quickEval" command
ctx.subscriptions.push(
commandRunnerWithProgress(
"codeQL.quickEvalContextEditor",
async (
progress: ProgressCallback,
token: CancellationToken,
uri: Uri | undefined,
) => await compileAndRunQuery(true, uri, progress, token, undefined),
{
title: "Running query",
cancellable: true,
},
// Open the query server logger on error since that's usually where the interesting errors appear.
queryServerLogger,
),
);
ctx.subscriptions.push(
commandRunnerWithProgress(
"codeQL.codeLensQuickEval",
@@ -1134,7 +1151,24 @@ async function activateWithInstalledDistribution(
),
);
// The "runVariantAnalysis" command is internal-only.
async function runVariantAnalysis(
progress: ProgressCallback,
token: CancellationToken,
uri: Uri | undefined,
): Promise<void> {
progress({
maxStep: 5,
step: 0,
message: "Getting credentials",
});
await variantAnalysisManager.runVariantAnalysis(
uri || window.activeTextEditor?.document.uri,
progress,
token,
);
}
ctx.subscriptions.push(
commandRunnerWithProgress(
"codeQL.runVariantAnalysis",
@@ -1142,19 +1176,23 @@ async function activateWithInstalledDistribution(
progress: ProgressCallback,
token: CancellationToken,
uri: Uri | undefined,
) => {
progress({
maxStep: 5,
step: 0,
message: "Getting credentials",
});
await variantAnalysisManager.runVariantAnalysis(
uri || window.activeTextEditor?.document.uri,
progress,
token,
);
) => await runVariantAnalysis(progress, token, uri),
{
title: "Run Variant Analysis",
cancellable: true,
},
),
);
// Since we are tracking extension usage through commands, this command mirrors the "codeQL.runVariantAnalysis" command
ctx.subscriptions.push(
commandRunnerWithProgress(
"codeQL.runVariantAnalysisContextEditor",
async (
progress: ProgressCallback,
token: CancellationToken,
uri: Uri | undefined,
) => await runVariantAnalysis(progress, token, uri),
{
title: "Run Variant Analysis",
cancellable: true,
@@ -1302,6 +1340,19 @@ async function activateWithInstalledDistribution(
commandRunner("codeQL.openReferencedFile", openReferencedFile),
);
// Since we are tracking extension usage through commands, this command mirrors the "codeQL.openReferencedFile" command
ctx.subscriptions.push(
commandRunner("codeQL.openReferencedFileContextEditor", openReferencedFile),
);
// Since we are tracking extension usage through commands, this command mirrors the "codeQL.openReferencedFile" command
ctx.subscriptions.push(
commandRunner(
"codeQL.openReferencedFileContextExplorer",
openReferencedFile,
),
);
ctx.subscriptions.push(
commandRunner("codeQL.previewQueryHelp", previewQueryHelp),
);
@@ -1480,6 +1531,22 @@ async function activateWithInstalledDistribution(
const cfgTemplateProvider = new TemplatePrintCfgProvider(cliServer, dbm);
ctx.subscriptions.push(astViewer);
async function viewAst(
progress: ProgressCallback,
token: CancellationToken,
selectedFile: Uri,
): Promise<void> {
const ast = await printAstTemplateProvider.provideAst(
progress,
token,
selectedFile ?? window.activeTextEditor?.document.uri,
);
if (ast) {
astViewer.updateRoots(await ast.getRoots(), ast.db, ast.fileName);
}
}
ctx.subscriptions.push(
commandRunnerWithProgress(
"codeQL.viewAst",
@@ -1487,16 +1554,39 @@ async function activateWithInstalledDistribution(
progress: ProgressCallback,
token: CancellationToken,
selectedFile: Uri,
) => {
const ast = await printAstTemplateProvider.provideAst(
progress,
token,
selectedFile ?? window.activeTextEditor?.document.uri,
);
if (ast) {
astViewer.updateRoots(await ast.getRoots(), ast.db, ast.fileName);
}
) => await viewAst(progress, token, selectedFile),
{
cancellable: true,
title: "Calculate AST",
},
),
);
// Since we are tracking extension usage through commands, this command mirrors the "codeQL.viewAst" command
ctx.subscriptions.push(
commandRunnerWithProgress(
"codeQL.viewAstContextExplorer",
async (
progress: ProgressCallback,
token: CancellationToken,
selectedFile: Uri,
) => await viewAst(progress, token, selectedFile),
{
cancellable: true,
title: "Calculate AST",
},
),
);
// Since we are tracking extension usage through commands, this command mirrors the "codeQL.viewAst" command
ctx.subscriptions.push(
commandRunnerWithProgress(
"codeQL.viewAstContextEditor",
async (
progress: ProgressCallback,
token: CancellationToken,
selectedFile: Uri,
) => await viewAst(progress, token, selectedFile),
{
cancellable: true,
title: "Calculate AST",
@@ -1522,6 +1612,44 @@ async function activateWithInstalledDistribution(
),
);
// Since we are tracking extension usage through commands, this command mirrors the "codeQL.viewCfg" command
ctx.subscriptions.push(
commandRunnerWithProgress(
"codeQL.viewCfgContextExplorer",
async (progress: ProgressCallback, token: CancellationToken) => {
const res = await cfgTemplateProvider.provideCfgUri(
window.activeTextEditor?.document,
);
if (res) {
await compileAndRunQuery(false, res[0], progress, token, undefined);
}
},
{
title: "Calculating Control Flow Graph",
cancellable: true,
},
),
);
// Since we are tracking extension usage through commands, this command mirrors the "codeQL.viewCfg" command
ctx.subscriptions.push(
commandRunnerWithProgress(
"codeQL.viewCfgContextEditor",
async (progress: ProgressCallback, token: CancellationToken) => {
const res = await cfgTemplateProvider.provideCfgUri(
window.activeTextEditor?.document,
);
if (res) {
await compileAndRunQuery(false, res[0], progress, token, undefined);
}
},
{
title: "Calculating Control Flow Graph",
cancellable: true,
},
),
);
const mockServer = new VSCodeMockGitHubApiServer(ctx);
ctx.subscriptions.push(mockServer);
ctx.subscriptions.push(

View File

@@ -1,4 +1,3 @@
import { dirname } from "path";
import { ensureFile } from "fs-extra";
import { DisposableObject } from "../pure/disposable-object";
@@ -13,10 +12,8 @@ import {
progress,
ProgressMessage,
WithProgressId,
compileQuery,
} from "../pure/legacy-messages";
import { ProgressCallback, ProgressTask } from "../commandRunner";
import { findQueryLogFile } from "../run-queries-shared";
import { ServerProcess } from "../json-rpc-server";
type WithProgressReporting = (
@@ -56,7 +53,7 @@ export class QueryServerClient extends DisposableObject {
this.queryServerStartListeners.push(e);
};
public activeQueryLogFile: string | undefined;
public activeQueryLogger: Logger;
constructor(
readonly config: QueryServerConfig,
@@ -65,6 +62,9 @@ export class QueryServerClient extends DisposableObject {
withProgressReporting: WithProgressReporting,
) {
super();
// Since no query is active when we initialize, just point the "active query logger" to the
// default logger.
this.activeQueryLogger = this.logger;
// When the query server configuration changes, restart the query server.
if (config.onDidChangeConfiguration !== undefined) {
this.push(
@@ -177,9 +177,8 @@ export class QueryServerClient extends DisposableObject {
args,
this.logger,
(data) =>
this.logger.log(data.toString(), {
this.activeQueryLogger.log(data.toString(), {
trailingNewline: false,
additionalLogLocation: this.activeQueryLogFile,
}),
undefined, // no listener for stdout
progressReporter,
@@ -240,8 +239,6 @@ export class QueryServerClient extends DisposableObject {
): Promise<R> {
const id = this.nextProgress++;
this.progressCallbacks[id] = progress;
this.updateActiveQuery(type.method, parameter);
try {
if (this.serverProcess === undefined) {
throw new Error("No query server process found.");
@@ -255,18 +252,4 @@ export class QueryServerClient extends DisposableObject {
delete this.progressCallbacks[id];
}
}
/**
* Updates the active query every time there is a new request to compile.
* The active query is used to specify the side log.
*
* This isn't ideal because in situations where there are queries running
* in parallel, each query's log messages are interleaved. Fixing this
* properly will require a change in the query server.
*/
private updateActiveQuery(method: string, parameter: any): void {
if (method === compileQuery.method) {
this.activeQueryLogFile = findQueryLogFile(dirname(parameter.resultPath));
}
}
}

View File

@@ -15,7 +15,7 @@ import {
} from "../helpers";
import { ProgressCallback } from "../commandRunner";
import { QueryMetadata } from "../pure/interface-types";
import { extLogger } from "../common";
import { extLogger, Logger, TeeLogger } from "../common";
import * as messages from "../pure/legacy-messages";
import { InitialQueryInfo, LocalQueryInfo } from "../query-results";
import * as qsClient from "./queryserver-client";
@@ -66,7 +66,8 @@ export class QueryInProgress {
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
queryInfo?: LocalQueryInfo,
logger: Logger,
queryInfo: LocalQueryInfo | undefined,
): Promise<messages.EvaluationResult> {
if (!dbItem.contents || dbItem.error) {
throw new Error("Can't run query on invalid database.");
@@ -137,7 +138,7 @@ export class QueryInProgress {
await this.queryEvalInfo.addQueryLogs(
queryInfo,
qs.cliServer,
qs.logger,
logger,
);
} else {
void showAndLogWarningMessage(
@@ -162,6 +163,7 @@ export class QueryInProgress {
program: messages.QlProgram,
progress: ProgressCallback,
token: CancellationToken,
logger: Logger,
): Promise<messages.CompilationMessage[]> {
let compiled: messages.CheckQueryResult | undefined;
try {
@@ -190,6 +192,11 @@ export class QueryInProgress {
target,
};
// Update the active query logger every time there is a new request to compile.
// This isn't ideal because in situations where there are queries running
// in parallel, each query's log messages are interleaved. Fixing this
// properly will require a change in the query server.
qs.activeQueryLogger = logger;
compiled = await qs.sendRequest(
messages.compileQuery,
params,
@@ -197,9 +204,7 @@ export class QueryInProgress {
progress,
);
} finally {
void qs.logger.log(" - - - COMPILATION DONE - - - ", {
additionalLogLocation: this.queryEvalInfo.logPath,
});
void logger.log(" - - - COMPILATION DONE - - - ");
}
return (compiled?.messages || []).filter(
(msg) => msg.severity === messages.Severity.ERROR,
@@ -386,6 +391,8 @@ export async function compileAndRunQueryAgainstDatabase(
metadata,
templates,
);
const logger = new TeeLogger(qs.logger, query.queryEvalInfo.logPath);
await query.queryEvalInfo.createTimestampFile();
let upgradeDir: tmp.DirectoryResult | undefined;
@@ -402,7 +409,7 @@ export async function compileAndRunQueryAgainstDatabase(
);
let errors;
try {
errors = await query.compile(qs, qlProgram, progress, token);
errors = await query.compile(qs, qlProgram, progress, token, logger);
} catch (e) {
if (
e instanceof ResponseError &&
@@ -422,6 +429,7 @@ export async function compileAndRunQueryAgainstDatabase(
dbItem,
progress,
token,
logger,
queryInfo,
);
if (result.resultType !== messages.QueryResultType.SUCCESS) {
@@ -439,18 +447,14 @@ export async function compileAndRunQueryAgainstDatabase(
result,
successful: result.resultType === messages.QueryResultType.SUCCESS,
logFileLocation: result.logFileLocation,
dispose: () => {
qs.logger.removeAdditionalLogLocation(result.logFileLocation);
},
};
} else {
// Error dialogs are limited in size and scrollability,
// so we include a general description of the problem,
// and direct the user to the output window for the detailed compilation messages.
// However we don't show quick eval errors there so we need to display them anyway.
void qs.logger.log(
void logger.log(
`Failed to compile query ${initialInfo.queryPath} against database scheme ${qlProgram.dbschemePath}:`,
{ additionalLogLocation: query.queryEvalInfo.logPath },
);
const formattedMessages: string[] = [];
@@ -459,9 +463,7 @@ export async function compileAndRunQueryAgainstDatabase(
const message = error.message || "[no error message available]";
const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
formattedMessages.push(formatted);
void qs.logger.log(formatted, {
additionalLogLocation: query.queryEvalInfo.logPath,
});
void logger.log(formatted);
}
if (initialInfo.isQuickEval && formattedMessages.length <= 2) {
// If there are more than 2 error messages, they will not be displayed well in a popup
@@ -484,9 +486,8 @@ export async function compileAndRunQueryAgainstDatabase(
try {
await upgradeDir?.cleanup();
} catch (e) {
void qs.logger.log(
void logger.log(
`Could not clean up the upgrades dir. Reason: ${getErrorMessage(e)}`,
{ additionalLogLocation: query.queryEvalInfo.logPath },
);
}
}
@@ -535,9 +536,6 @@ function createSyntheticResult(
runId: 0,
},
successful: false,
dispose: () => {
/**/
},
};
}

View File

@@ -794,74 +794,66 @@ export class DatabaseManager extends DisposableObject {
}
public async loadPersistedState(): Promise<void> {
return withProgress(
{
location: vscode.ProgressLocation.Notification,
},
async (progress, token) => {
const currentDatabaseUri =
this.ctx.workspaceState.get<string>(CURRENT_DB);
const databases = this.ctx.workspaceState.get<PersistedDatabaseItem[]>(
DB_LIST,
[],
return withProgress(async (progress, token) => {
const currentDatabaseUri =
this.ctx.workspaceState.get<string>(CURRENT_DB);
const databases = this.ctx.workspaceState.get<PersistedDatabaseItem[]>(
DB_LIST,
[],
);
let step = 0;
progress({
maxStep: databases.length,
message: "Loading persisted databases",
step,
});
try {
void this.logger.log(
`Found ${databases.length} persisted databases: ${databases
.map((db) => db.uri)
.join(", ")}`,
);
let step = 0;
progress({
maxStep: databases.length,
message: "Loading persisted databases",
step,
});
try {
void this.logger.log(
`Found ${databases.length} persisted databases: ${databases
.map((db) => db.uri)
.join(", ")}`,
);
for (const database of databases) {
progress({
maxStep: databases.length,
message: `Loading ${
database.options?.displayName || "databases"
}`,
step: ++step,
});
for (const database of databases) {
progress({
maxStep: databases.length,
message: `Loading ${database.options?.displayName || "databases"}`,
step: ++step,
});
const databaseItem =
await this.createDatabaseItemFromPersistedState(
progress,
token,
database,
);
try {
await databaseItem.refresh();
await this.registerDatabase(progress, token, databaseItem);
if (currentDatabaseUri === database.uri) {
await this.setCurrentDatabaseItem(databaseItem, true);
}
void this.logger.log(
`Loaded database ${databaseItem.name} at URI ${database.uri}.`,
);
} catch (e) {
// When loading from persisted state, leave invalid databases in the list. They will be
// marked as invalid, and cannot be set as the current database.
void this.logger.log(
`Error loading database ${database.uri}: ${e}.`,
);
const databaseItem = await this.createDatabaseItemFromPersistedState(
progress,
token,
database,
);
try {
await databaseItem.refresh();
await this.registerDatabase(progress, token, databaseItem);
if (currentDatabaseUri === database.uri) {
await this.setCurrentDatabaseItem(databaseItem, true);
}
void this.logger.log(
`Loaded database ${databaseItem.name} at URI ${database.uri}.`,
);
} catch (e) {
// When loading from persisted state, leave invalid databases in the list. They will be
// marked as invalid, and cannot be set as the current database.
void this.logger.log(
`Error loading database ${database.uri}: ${e}.`,
);
}
await this.updatePersistedDatabaseList();
} catch (e) {
// database list had an unexpected type - nothing to be done?
void showAndLogExceptionWithTelemetry(
redactableError(
asError(e),
)`Database list loading failed: ${getErrorMessage(e)}`,
);
}
await this.updatePersistedDatabaseList();
} catch (e) {
// database list had an unexpected type - nothing to be done?
void showAndLogExceptionWithTelemetry(
redactableError(
asError(e),
)`Database list loading failed: ${getErrorMessage(e)}`,
);
}
void this.logger.log("Finished loading persisted databases.");
},
);
void this.logger.log("Finished loading persisted databases.");
});
}
public get databaseItems(): readonly DatabaseItem[] {

View File

@@ -126,6 +126,7 @@ export interface RunQueryParams {
singletonExternalInputs: Record<string, string>;
dilPath?: string;
logPath?: string;
extensionPacks?: string[];
}
export interface RunQueryResult {

View File

@@ -111,9 +111,14 @@ export class HistoryTreeDataProvider
return "remoteResultsItem";
}
case QueryStatus.Failed:
return element.t === "local"
? "cancelledResultsItem"
: "cancelledRemoteResultsItem";
if (element.t === "local") {
return "cancelledResultsItem";
} else if (element.variantAnalysis.actionsWorkflowRunId === undefined) {
return "cancelledRemoteResultsItemWithoutLogs";
} else {
return "cancelledRemoteResultsItem";
}
default:
assertNever(element.status);
}

View File

@@ -556,7 +556,6 @@ export class QueryHistoryManager extends DisposableObject {
!(await pathExists(item.completedQuery?.query.querySaveDir))
) {
this.treeDataProvider.remove(item);
item.completedQuery?.dispose();
}
}),
);
@@ -577,7 +576,6 @@ export class QueryHistoryManager extends DisposableObject {
// Removing in progress local queries is not supported. They must be cancelled first.
if (item.status !== QueryStatus.InProgress) {
this.treeDataProvider.remove(item);
item.completedQuery?.dispose();
// User has explicitly asked for this query to be removed.
// We need to delete it from disk as well.

View File

@@ -55,11 +55,6 @@ export class CompletedQueryInfo implements QueryWithResults {
readonly logFileLocation?: string;
resultCount: number;
/**
* This dispose method is called when the query is removed from the history view.
*/
dispose: () => void;
/**
* Map from result set name to SortedResultSetInfo.
*/
@@ -85,10 +80,6 @@ export class CompletedQueryInfo implements QueryWithResults {
this.message = evaluation.message;
this.successful = evaluation.successful;
// Use the dispose method from the evaluation.
// The dispose will clean up any additional log locations that this
// query may have created.
this.dispose = evaluation.dispose;
this.sortedResultsInfo = {};
this.resultCount = 0;

View File

@@ -55,10 +55,6 @@ export async function deserializeQueryHistory(
q.completedQuery.query,
QueryEvaluationInfo.prototype,
);
// deserialized queries do not need to be disposed
q.completedQuery.dispose = () => {
/**/
};
// Previously, there was a typo in the completedQuery type. There was a field
// `sucessful` and it was renamed to `successful`. We need to handle this case.

View File

@@ -1,4 +1,3 @@
import { dirname } from "path";
import { ensureFile } from "fs-extra";
import { DisposableObject } from "../pure/disposable-object";
@@ -12,9 +11,7 @@ import {
ProgressMessage,
WithProgressId,
} from "../pure/new-messages";
import * as messages from "../pure/new-messages";
import { ProgressCallback, ProgressTask } from "../commandRunner";
import { findQueryLogFile } from "../run-queries-shared";
import { ServerProcess } from "../json-rpc-server";
type ServerOpts = {
@@ -53,7 +50,7 @@ export class QueryServerClient extends DisposableObject {
this.queryServerStartListeners.push(e);
};
public activeQueryLogFile: string | undefined;
public activeQueryLogger: Logger;
constructor(
readonly config: QueryServerConfig,
@@ -62,6 +59,9 @@ export class QueryServerClient extends DisposableObject {
withProgressReporting: WithProgressReporting,
) {
super();
// Since no query is active when we initialize, just point the "active query logger" to the
// default logger.
this.activeQueryLogger = this.logger;
// When the query server configuration changes, restart the query server.
if (config.onDidChangeConfiguration !== undefined) {
this.push(
@@ -167,9 +167,8 @@ export class QueryServerClient extends DisposableObject {
args,
this.logger,
(data) =>
this.logger.log(data.toString(), {
this.activeQueryLogger.log(data.toString(), {
trailingNewline: false,
additionalLogLocation: this.activeQueryLogFile,
}),
undefined, // no listener for stdout
progressReporter,
@@ -210,7 +209,6 @@ export class QueryServerClient extends DisposableObject {
const id = this.nextProgress++;
this.progressCallbacks[id] = progress;
this.updateActiveQuery(type.method, parameter);
try {
if (this.serverProcess === undefined) {
throw new Error("No query server process found.");
@@ -224,20 +222,4 @@ export class QueryServerClient extends DisposableObject {
delete this.progressCallbacks[id];
}
}
/**
* Updates the active query every time there is a new request to compile.
* The active query is used to specify the side log.
*
* This isn't ideal because in situations where there are queries running
* in parallel, each query's log messages are interleaved. Fixing this
* properly will require a change in the query server.
*/
private updateActiveQuery(method: string, parameter: any): void {
if (method === messages.runQuery.method) {
this.activeQueryLogFile = findQueryLogFile(
dirname(dirname((parameter as messages.RunQueryParams).outputPath)),
);
}
}
}

View File

@@ -9,7 +9,7 @@ import {
showAndLogWarningMessage,
tryGetQueryMetadata,
} from "../helpers";
import { extLogger } from "../common";
import { extLogger, TeeLogger } from "../common";
import * as messages from "../pure/new-messages";
import { QueryResultType } from "../pure/legacy-messages";
import { InitialQueryInfo, LocalQueryInfo } from "../query-results";
@@ -70,6 +70,10 @@ export async function compileAndRunQueryAgainstDatabase(
: { query: {} };
const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
const extensionPacks = (await qs.cliServer.useExtensionPacks())
? Object.keys(await qs.cliServer.resolveQlpacks(diskWorkspaceFolders, true))
: undefined;
const db = dbItem.databaseUri.fsPath;
const logPath = queryInfo ? query.evalLogPath : undefined;
const queryToRun: messages.RunQueryParams = {
@@ -82,10 +86,17 @@ export async function compileAndRunQueryAgainstDatabase(
dilPath: query.dilPath,
logPath,
target,
extensionPacks,
};
const logger = new TeeLogger(qs.logger, query.logPath);
await query.createTimestampFile();
let result: messages.RunQueryResult | undefined;
try {
// Update the active query logger every time there is a new request to compile.
// This isn't ideal because in situations where there are queries running
// in parallel, each query's log messages are interleaved. Fixing this
// properly will require a change in the query server.
qs.activeQueryLogger = logger;
result = await qs.sendRequest(
messages.runQuery,
queryToRun,
@@ -100,7 +111,7 @@ export async function compileAndRunQueryAgainstDatabase(
} finally {
if (queryInfo) {
if (await query.hasEvalLog()) {
await query.addQueryLogs(queryInfo, qs.cliServer, qs.logger);
await query.addQueryLogs(queryInfo, qs.cliServer, logger);
} else {
void showAndLogWarningMessage(
`Failed to write structured evaluator log to ${query.evalLogPath}.`,
@@ -155,8 +166,5 @@ export async function compileAndRunQueryAgainstDatabase(
},
message,
successful,
dispose: () => {
qs.logger.removeAdditionalLogLocation(undefined);
},
};
}

View File

@@ -298,12 +298,8 @@ export class QueryEvaluationInfo {
this.evalLogEndSummaryPath,
"utf-8",
);
void logger.log(" --- Evaluator Log Summary --- ", {
additionalLogLocation: this.logPath,
});
void logger.log(endSummaryContent, {
additionalLogLocation: this.logPath,
});
void logger.log(" --- Evaluator Log Summary --- ");
void logger.log(endSummaryContent);
} catch (e) {
void showAndLogWarningMessage(
`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`,
@@ -436,7 +432,6 @@ export class QueryEvaluationInfo {
export interface QueryWithResults {
readonly query: QueryEvaluationInfo;
readonly logFileLocation?: string;
readonly dispose: () => void;
readonly successful?: boolean;
readonly message?: string;
readonly result: legacyMessages.EvaluationResult;

View File

@@ -1,5 +1,5 @@
[
"v2.12.3",
"v2.12.4",
"v2.11.6",
"v2.7.6",
"v2.8.5",

View File

@@ -4,6 +4,5 @@ export function createMockLogger(): Logger {
return {
log: jest.fn(() => Promise.resolve()),
show: jest.fn(),
removeAdditionalLogLocation: jest.fn(),
};
}

View File

@@ -1,7 +1,7 @@
import { readdirSync, readFileSync } from "fs-extra";
import { join } from "path";
import * as tmp from "tmp";
import { OutputChannelLogger } from "../../../src/common";
import { Logger, OutputChannelLogger, TeeLogger } from "../../../src/common";
jest.setTimeout(999999);
@@ -58,16 +58,19 @@ describe("OutputChannelLogger tests", function () {
expect(mockOutputChannel.appendLine).not.toBeCalledWith("yyy");
expect(mockOutputChannel.append).toBeCalledWith("yyy");
await logger.log("zzz", createLogOptions("hucairz"));
const hucairz = createSideLogger(logger, "hucairz");
await hucairz.log("zzz");
// should have created 1 side log
expect(readdirSync(tempFolders.storagePath.name)).toEqual(["hucairz"]);
});
it("should create a side log", async () => {
await logger.log("xxx", createLogOptions("first"));
await logger.log("yyy", createLogOptions("second"));
await logger.log("zzz", createLogOptions("first", false));
const first = createSideLogger(logger, "first");
await first.log("xxx");
const second = createSideLogger(logger, "second");
await second.log("yyy");
await first.log("zzz", { trailingNewline: false });
await logger.log("aaa");
// expect 2 side logs
@@ -82,16 +85,13 @@ describe("OutputChannelLogger tests", function () {
).toBe("yyy\n");
});
function createLogOptions(
function createSideLogger(
logger: Logger,
additionalLogLocation: string,
trailingNewline?: boolean,
) {
return {
additionalLogLocation: join(
tempFolders.storagePath.name,
additionalLogLocation,
),
trailingNewline,
};
): Logger {
return new TeeLogger(
logger,
join(tempFolders.storagePath.name, additionalLogLocation),
);
}
});

View File

@@ -0,0 +1,8 @@
extensions:
- data:
- [2]
- [3]
- [4]
addsTo:
extensible: testExtensible
pack: semmle/has-extension

View File

@@ -0,0 +1,8 @@
name: semmle/targets-extension
library: true
version: 0.0.0
extensionTargets:
semmle/has-extension: '*'
dataExtensions:
- ext/*

View File

@@ -0,0 +1,7 @@
extensions:
- data:
- [1]
addsTo:
extensible: testExtensible
pack: semmle/has-extension

View File

@@ -0,0 +1,6 @@
name: semmle/has-extension
version: 0.0.0
dependencies:
codeql/javascript-all: '*'
dataExtensions:
- ext/*

View File

@@ -0,0 +1,8 @@
import javascript
extensible predicate testExtensible(int i);
from int i
where testExtensible(i)
select i

View File

@@ -74,7 +74,6 @@ export function createMockQueryWithResults({
hasInterpretedResults?: boolean;
hasMetadata?: boolean;
}): QueryWithResults {
const dispose = jest.fn();
const deleteQuery = jest.fn();
const metadata = hasMetadata
? ({ name: "query-name" } as QueryMetadata)
@@ -87,7 +86,6 @@ export function createMockQueryWithResults({
metadata,
} as unknown as QueryEvaluationInfo,
successful: didRunSuccessfully,
dispose,
result: {
evaluationTime: 1,
queryId: 0,

View File

@@ -18,6 +18,7 @@ const config = {
"--disable-extension",
"github.copilot",
path.resolve(rootDir, "test/data"),
path.resolve(rootDir, "test/data-extensions"), // folder containing the extension packs and packs that are targeted by the extension pack
// CLI integration tests requires a multi-root workspace so that the data and the QL sources are accessible.
...(process.env.TEST_CODEQL_PATH ? [process.env.TEST_CODEQL_PATH] : []),
],

View File

@@ -61,6 +61,7 @@ describe("Packaging commands", () => {
);
await handleDownloadPacks(cli, progress);
expect(showAndLogExceptionWithTelemetrySpy).not.toHaveBeenCalled();
expect(showAndLogInformationMessageSpy).toHaveBeenCalledWith(
expect.stringContaining("Finished downloading packs."),
);
@@ -73,6 +74,7 @@ describe("Packaging commands", () => {
inputBoxSpy.mockResolvedValue("codeql/csharp-solorigate-queries");
await handleDownloadPacks(cli, progress);
expect(showAndLogExceptionWithTelemetrySpy).not.toHaveBeenCalled();
expect(showAndLogInformationMessageSpy).toHaveBeenCalledWith(
expect.stringContaining("Finished downloading packs."),
);

View File

@@ -19,11 +19,13 @@ import { DatabaseItem, DatabaseManager } from "../../../src/local-databases";
import { CodeQLExtensionInterface } from "../../../src/extension";
import { cleanDatabases, dbLoc, storagePath } from "../global.helper";
import { importArchiveDatabase } from "../../../src/databaseFetcher";
import { CodeQLCliServer } from "../../../src/cli";
import { CliVersionConstraint, CodeQLCliServer } from "../../../src/cli";
import { describeWithCodeQL } from "../cli";
import { tmpDir } from "../../../src/helpers";
import { createInitialQueryInfo } from "../../../src/run-queries-shared";
import { QueryRunner } from "../../../src/queryRunner";
import { CompletedQueryInfo } from "../../../src/query-results";
import { SELECT_QUERY_NAME } from "../../../src/contextual/locationFinder";
jest.setTimeout(20_000);
@@ -96,6 +98,78 @@ describeWithCodeQL()("Queries", () => {
await cleanDatabases(databaseManager);
});
describe("extension packs", () => {
const queryUsingExtensionPath = join(
__dirname,
"../..",
"data-extensions",
"pack-using-extensions",
"query.ql",
);
it("should run a query that has an extension without looking for extensions in the workspace", async () => {
if (!(await supportsExtensionPacks())) {
console.log(
`Skipping test because it is only supported for CodeQL CLI versions >= ${CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND}`,
);
return;
}
await cli.setUseExtensionPacks(false);
const parsedResults = await runQueryWithExtensions();
expect(parsedResults).toEqual([1]);
});
it("should run a query that has an extension and look for extensions in the workspace", async () => {
if (!(await supportsExtensionPacks())) {
return;
}
await cli.setUseExtensionPacks(true);
const parsedResults = await runQueryWithExtensions();
expect(parsedResults).toEqual([1, 2, 3, 4]);
});
async function supportsExtensionPacks(): Promise<boolean> {
if (await qs.cliServer.cliConstraints.supportsQlpacksKind()) {
return true;
}
console.log(
`Skipping test because it is only supported for CodeQL CLI versions >= ${CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND}`,
);
return false;
}
async function runQueryWithExtensions() {
const result = new CompletedQueryInfo(
await qs.compileAndRunQueryAgainstDatabase(
dbItem,
await mockInitialQueryInfo(queryUsingExtensionPath),
join(tmpDir.name, "mock-storage-path"),
progress,
token,
),
);
// Check that query was successful
expect(result.successful).toBe(true);
// Load query results
const chunk = await qs.cliServer.bqrsDecode(
result.getResultsPath(SELECT_QUERY_NAME, true),
SELECT_QUERY_NAME,
{
// there should only be one result
offset: 0,
pageSize: 10,
},
);
// Extract the results as an array.
return chunk.tuples.map((t) => t[0]);
}
});
it("should run a query", async () => {
const queryPath = join(__dirname, "data", "simple-query.ql");
const result = qs.compileAndRunQueryAgainstDatabase(

View File

@@ -19,7 +19,10 @@ import {
storagePath,
} from "../../global.helper";
import { VariantAnalysisResultsManager } from "../../../../src/variant-analysis/variant-analysis-results-manager";
import { VariantAnalysisStatus } from "../../../../src/variant-analysis/shared/variant-analysis";
import {
VariantAnalysisStatus,
VariantAnalysisSubmission,
} from "../../../../src/variant-analysis/shared/variant-analysis";
import { VariantAnalysis as VariantAnalysisApiResponse } from "../../../../src/variant-analysis/gh-api/variant-analysis";
import { createMockApiResponse } from "../../../factories/variant-analysis/gh-api/variant-analysis-api-response";
import { UserCancellationException } from "../../../../src/commandRunner";
@@ -28,6 +31,9 @@ import { DbManager } from "../../../../src/databases/db-manager";
import { ExtensionApp } from "../../../../src/common/vscode/vscode-app";
import { DbConfigStore } from "../../../../src/databases/config/db-config-store";
import { mockedQuickPickItem } from "../../utils/mocking.helpers";
import { QueryLanguage } from "../../../../src/common/query-language";
import { readBundledPack } from "../../utils/bundled-pack-helpers";
import { load } from "js-yaml";
// up to 3 minutes per test
jest.setTimeout(3 * 60 * 1000);
@@ -81,10 +87,6 @@ describe("Variant Analysis Manager", () => {
"data-remote-qlpack/qlpack.yml",
).fsPath;
function getFile(file: string): Uri {
return Uri.file(join(baseDir, file));
}
beforeEach(async () => {
jest
.spyOn(window, "showQuickPick")
@@ -202,5 +204,136 @@ describe("Variant Analysis Manager", () => {
await expect(promise).rejects.toThrow(UserCancellationException);
});
describe("check variant analysis generated packs", () => {
beforeEach(() => {
mockSubmitVariantAnalysis = jest
.spyOn(ghApiClient, "submitVariantAnalysis")
.mockResolvedValue({
id: 1,
query_language: QueryLanguage.Javascript,
query_pack_url: "http://example.com",
created_at: "2021-01-01T00:00:00Z",
updated_at: "2021-01-01T00:00:00Z",
status: "in_progress",
controller_repo: {
id: 1,
name: "vscode-codeql",
full_name: "github/vscode-codeql",
private: false,
},
actions_workflow_run_id: 20,
scanned_repositories: [] as any[],
});
executeCommandSpy = jest.spyOn(commands, "executeCommand");
});
it("should run a remote query that is part of a qlpack", async () => {
await doVariantAnalysisTest({
queryPath: "data-remote-qlpack/in-pack.ql",
filesThatExist: ["in-pack.ql", "lib.qll"],
filesThatDoNotExist: [],
qlxFilesThatExist: ["in-pack.qlx"],
});
});
it("should run a remote query that is not part of a qlpack", async () => {
await doVariantAnalysisTest({
queryPath: "data-remote-no-qlpack/in-pack.ql",
filesThatExist: ["in-pack.ql"],
filesThatDoNotExist: ["lib.qll", "not-in-pack.ql"],
qlxFilesThatExist: ["in-pack.qlx"],
});
});
it("should run a remote query that is nested inside a qlpack", async () => {
await doVariantAnalysisTest({
queryPath: "data-remote-qlpack-nested/subfolder/in-pack.ql",
filesThatExist: ["subfolder/in-pack.ql", "otherfolder/lib.qll"],
filesThatDoNotExist: ["subfolder/not-in-pack.ql"],
qlxFilesThatExist: ["subfolder/in-pack.qlx"],
});
});
});
async function doVariantAnalysisTest({
queryPath,
filesThatExist,
qlxFilesThatExist,
filesThatDoNotExist,
}: {
queryPath: string;
filesThatExist: string[];
qlxFilesThatExist: string[];
filesThatDoNotExist: string[];
}) {
const fileUri = getFile(queryPath);
await variantAnalysisManager.runVariantAnalysis(
fileUri,
progress,
cancellationTokenSource.token,
);
expect(mockSubmitVariantAnalysis).toBeCalledTimes(1);
expect(executeCommandSpy).toBeCalledWith(
"codeQL.monitorVariantAnalysis",
expect.objectContaining({
query: expect.objectContaining({ filePath: fileUri.fsPath }),
}),
);
const request: VariantAnalysisSubmission =
mockSubmitVariantAnalysis.mock.calls[0][1];
const packFS = await readBundledPack(request.query.pack);
filesThatExist.forEach((file) => {
expect(packFS.fileExists(file)).toBe(true);
});
if (await cli.cliConstraints.supportsQlxRemote()) {
qlxFilesThatExist.forEach((file) => {
expect(packFS.fileExists(file)).toBe(true);
});
}
filesThatDoNotExist.forEach((file) => {
expect(packFS.fileExists(file)).toBe(false);
});
expect(
packFS.fileExists("qlpack.yml") || packFS.fileExists("codeql-pack.yml"),
).toBe(true);
// depending on the cli version, we should have one of these files
expect(
packFS.fileExists("qlpack.lock.yml") ||
packFS.fileExists("codeql-pack.lock.yml"),
).toBe(true);
const packFileName = packFS.fileExists("qlpack.yml")
? "qlpack.yml"
: "codeql-pack.yml";
const qlpackContents = load(
packFS.fileContents(packFileName).toString("utf-8"),
);
expect(qlpackContents.name).toEqual("codeql-remote/query");
expect(qlpackContents.version).toEqual("0.0.0");
expect(qlpackContents.dependencies?.["codeql/javascript-all"]).toEqual(
"*",
);
const qlpackLockContents = load(
packFS.fileContents("codeql-pack.lock.yml").toString("utf-8"),
);
const actualLockKeys = Object.keys(qlpackLockContents.dependencies);
// The lock file should contain at least codeql/javascript-all.
expect(actualLockKeys).toContain("codeql/javascript-all");
}
function getFile(file: string): Uri {
return Uri.file(join(baseDir, file));
}
});
});

View File

@@ -344,7 +344,6 @@ describe("QueryHistoryManager", () => {
});
it("should remove the item", () => {
expect(toDelete.completedQuery!.dispose).toBeCalledTimes(1);
expect(queryHistoryManager.treeDataProvider.allHistory).toEqual(
expect.not.arrayContaining([toDelete]),
);
@@ -387,7 +386,6 @@ describe("QueryHistoryManager", () => {
});
it("should remove the item", () => {
expect(toDelete.completedQuery!.dispose).toBeCalledTimes(1);
expect(queryHistoryManager.treeDataProvider.allHistory).toEqual(
expect.not.arrayContaining([toDelete]),
);

View File

@@ -470,7 +470,6 @@ describe("query-results", () => {
query: query.queryEvalInfo,
successful: didRunSuccessfully,
message: "foo",
dispose: jest.fn(),
result: {
evaluationTime: 1,
queryId: 0,

View File

@@ -259,7 +259,6 @@ describe("serialize and deserialize", () => {
query: query.queryEvalInfo,
successful: didRunSuccessfully,
message: "foo",
dispose: jest.fn(),
result: {
evaluationTime: 1,
queryId: 0,

View File

@@ -190,6 +190,7 @@ describe("run-queries", () => {
mockQlProgram,
mockProgress as any,
mockCancel as any,
qs.logger,
);
expect(results).toEqual([{ message: "err", severity: Severity.ERROR }]);