mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

QueryServer: Abstract over the query running parts of the query server in preperation for the new query server.

Browse Source
This commit is contained in:
alexet
2022-08-25 17:40:21 +01:00
committed by Andrew Eisenberg
parent 45efca9425
commit e648d9c67c
30 changed files with 1723 additions and 1460 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
extensions/ql-vscode/src/cli.ts
View File

@@ -16,7 +16,7 @@ import { DistributionProvider, FindDistributionResultKind } from './distribution
import { assertNever, getErrorMessage, getErrorStack } from './pure/helpers-pure';
import { QueryMetadata, SortDirection } from './pure/interface-types';
import { Logger, ProgressReporter } from './logging';
import { CompilationMessage } from './pure/messages';
import { CompilationMessage } from './pure/legacy-messages';
import { sarifParser } from './sarif-parser';
import { dbSchemeToLanguage, walkDirectory } from './helpers';
@@ -1248,6 +1248,9 @@ export class CliVersionConstraint {
*/
public static CLI_VERSION_WITH_LANGUAGE = new SemVer('2.4.1');
public static CLI_VERSION_WITH_NONDESTURCTIVE_UPGRADES = new SemVer('2.4.2');
/**
* CLI version where `codeql resolve upgrades` supports
* the `--allow-downgrades` flag
@@ -1264,14 +1267,6 @@ export class CliVersionConstraint {
*/
public static CLI_VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
/**
* CLI version where non destructive upgrades were introduced.
*
* This was landed in multiple parts so this is the version where all necessary feature were supported.
*/
public static CLI_VERSION_WITH_NON_DESTRUCTIVE_UPGRADES = new SemVer('2.4.2');
/**
* CLI version where the `--allow-library-packs` option to `codeql resolve queries` was
* introduced.
@@ -1351,6 +1346,10 @@ export class CliVersionConstraint {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_LANGUAGE);
}
public async supportsNonDestructiveUpgrades() {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NONDESTURCTIVE_UPGRADES);
}
public async supportsDowngrades() {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DOWNGRADES);
}
@@ -1367,10 +1366,6 @@ export class CliVersionConstraint {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DB_REGISTRATION);
}
async supportsNonDestructiveUpgrades(): Promise<boolean> {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NON_DESTRUCTIVE_UPGRADES);
}
async supportsDatabaseUnbundle() {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DATABASE_UNBUNDLE);
}

2
extensions/ql-vscode/src/contextual/astBuilder.ts
View File

@@ -1,10 +1,10 @@
import { QueryWithResults } from '../run-queries';
import { CodeQLCliServer } from '../cli';
import { DecodedBqrsChunk, BqrsId, EntityValue } from '../pure/bqrs-cli-types';
import { DatabaseItem } from '../databases';
import { ChildAstItem, AstItem } from '../astViewer';
import fileRangeFromURI from './fileRangeFromURI';
import { Uri } from 'vscode';
import { QueryWithResults } from '../run-queries-shared';
/**
* A class that wraps a tree of QL results from a query that

32
extensions/ql-vscode/src/contextual/locationFinder.ts
View File

@@ -3,13 +3,12 @@ import { ColumnKindCode, EntityValue, getResultSetSchema, ResultSetSchema } from
import { CodeQLCliServer } from '../cli';
import { DatabaseManager, DatabaseItem } from '../databases';
import fileRangeFromURI from './fileRangeFromURI';
import * as messages from '../pure/messages';
import { QueryServerClient } from '../queryserver-client';
import { QueryWithResults, compileAndRunQueryAgainstDatabase, createInitialQueryInfo } from '../run-queries';
import { ProgressCallback } from '../commandRunner';
import { KeyType } from './keyType';
import { qlpackOfDatabase, resolveQueries } from './queryResolver';
import { CancellationToken, LocationLink, Uri } from 'vscode';
import { createInitialQueryInfo, QueryWithResults } from '../run-queries-shared';
import { QueryRunner } from '../queryRunner';
export const SELECT_QUERY_NAME = '#select';
export const TEMPLATE_NAME = 'selectedSourceFile';
@@ -35,7 +34,7 @@ export interface FullLocationLink extends LocationLink {
*/
export async function getLocationsForUriString(
cli: CodeQLCliServer,
qs: QueryServerClient,
qs: QueryRunner,
dbm: DatabaseManager,
uriString: string,
keyType: KeyType,
@@ -65,19 +64,8 @@ export async function getLocationsForUriString(
},
false
);
const results = await compileAndRunQueryAgainstDatabase(
cli,
qs,
db,
initialInfo,
queryStorageDir,
progress,
token,
templates
);
if (results.result.resultType == messages.QueryResultType.SUCCESS) {
const results = await qs.compileAndRunQueryAgainstDatabase(db, initialInfo, queryStorageDir, progress, token, templates);
if (results.sucessful) {
links.push(...await getLinksFromResults(results, cli, db, filter));
}
}
@@ -114,15 +102,9 @@ async function getLinksFromResults(
return localLinks;
}
function createTemplates(path: string): messages.TemplateDefinitions {
function createTemplates(path: string): Record<string, string> {
return {
[TEMPLATE_NAME]: {
values: {
tuples: [[{
stringValue: path
}]]
}
}
[TEMPLATE_NAME]: path
};
}

44
extensions/ql-vscode/src/contextual/templateProvider.ts
View File

@@ -16,9 +16,6 @@ import { CodeQLCliServer } from '../cli';
import { DatabaseManager } from '../databases';
import { CachedOperation } from '../helpers';
import { ProgressCallback, withProgress } from '../commandRunner';
import * as messages from '../pure/messages';
import { QueryServerClient } from '../queryserver-client';
import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo, QueryWithResults } from '../run-queries';
import AstBuilder from './astBuilder';
import {
KeyType,
@@ -26,6 +23,8 @@ import {
import { FullLocationLink, getLocationsForUriString, TEMPLATE_NAME } from './locationFinder';
import { qlpackOfDatabase, resolveQueries } from './queryResolver';
import { isCanary, NO_CACHE_AST_VIEWER } from '../config';
import { createInitialQueryInfo, QueryWithResults } from '../run-queries-shared';
import { QueryRunner } from '../queryRunner';
/**
* Run templated CodeQL queries to find definitions and references in
@@ -39,7 +38,7 @@ export class TemplateQueryDefinitionProvider implements DefinitionProvider {
constructor(
private cli: CodeQLCliServer,
private qs: QueryServerClient,
private qs: QueryRunner,
private dbm: DatabaseManager,
private queryStorageDir: string,
) {
@@ -83,7 +82,7 @@ export class TemplateQueryReferenceProvider implements ReferenceProvider {
constructor(
private cli: CodeQLCliServer,
private qs: QueryServerClient,
private qs: QueryRunner,
private dbm: DatabaseManager,
private queryStorageDir: string,
) {
@@ -137,7 +136,7 @@ export class TemplatePrintAstProvider {
constructor(
private cli: CodeQLCliServer,
private qs: QueryServerClient,
private qs: QueryRunner,
private dbm: DatabaseManager,
private queryStorageDir: string,
) {
@@ -195,14 +194,9 @@ export class TemplatePrintAstProvider {
}
const query = queries[0];
const templates: messages.TemplateDefinitions = {
[TEMPLATE_NAME]: {
values: {
tuples: [[{
stringValue: zippedArchive.pathWithinSourceArchive
}]]
}
}
const templates: Record<string, string> = {
[TEMPLATE_NAME]:
zippedArchive.pathWithinSourceArchive
};
const initialInfo = await createInitialQueryInfo(
@@ -215,9 +209,7 @@ export class TemplatePrintAstProvider {
);
return {
query: await compileAndRunQueryAgainstDatabase(
this.cli,
this.qs,
query: await this.qs.compileAndRunQueryAgainstDatabase(
db,
initialInfo,
this.queryStorageDir,
@@ -231,23 +223,23 @@ export class TemplatePrintAstProvider {
}
export class TemplatePrintCfgProvider {
private cache: CachedOperation<[Uri, messages.TemplateDefinitions] | undefined>;
private cache: CachedOperation<[Uri, Record<string, string>] | undefined>;
constructor(
private cli: CodeQLCliServer,
private dbm: DatabaseManager,
) {
this.cache = new CachedOperation<[Uri, messages.TemplateDefinitions] | undefined>(this.getCfgUri.bind(this));
this.cache = new CachedOperation<[Uri, Record<string, string>] | undefined>(this.getCfgUri.bind(this));
}
async provideCfgUri(document?: TextDocument): Promise<[Uri, messages.TemplateDefinitions] | undefined> {
async provideCfgUri(document?: TextDocument): Promise<[Uri, Record<string, string>] | undefined> {
if (!document) {
return;
}
return await this.cache.get(document.uri.toString());
}
private async getCfgUri(uriString: string): Promise<[Uri, messages.TemplateDefinitions]> {
private async getCfgUri(uriString: string): Promise<[Uri, Record<string, string>]> {
const uri = Uri.parse(uriString, true);
if (uri.scheme !== zipArchiveScheme) {
throw new Error('CFG Viewing is only available for databases with zipped source archives.');
@@ -275,14 +267,8 @@ export class TemplatePrintCfgProvider {
const queryUri = Uri.file(queries[0]);
const templates: messages.TemplateDefinitions = {
[TEMPLATE_NAME]: {
values: {
tuples: [[{
stringValue: zippedArchive.pathWithinSourceArchive
}]]
}
}
const templates: Record<string, string> = {
[TEMPLATE_NAME]: zippedArchive.pathWithinSourceArchive
};
return [queryUri, templates];

20
extensions/ql-vscode/src/databases-ui.ts
View File

@@ -28,9 +28,6 @@ import {
showAndLogErrorMessage
} from './helpers';
import { logger } from './logging';
import { clearCacheInDatabase } from './run-queries';
import * as qsClient from './queryserver-client';
import { upgradeDatabaseExplicit } from './upgrades';
import {
importArchiveDatabase,
promptImportGithubDatabase,
@@ -40,6 +37,7 @@ import {
import { CancellationToken } from 'vscode';
import { asyncFilter, getErrorMessage } from './pure/helpers-pure';
import { Credentials } from './authentication';
import { QueryRunner } from './queryRunner';
import { isCanary } from './config';
type ThemableIconPath = { light: string; dark: string } | string;
@@ -220,7 +218,7 @@ export class DatabaseUI extends DisposableObject {
public constructor(
private databaseManager: DatabaseManager,
private readonly queryServer: qsClient.QueryServerClient | undefined,
private readonly queryServer: QueryRunner | undefined,
private readonly storagePath: string,
readonly extensionPath: string,
private readonly getCredentials: () => Promise<Credentials>
@@ -390,9 +388,9 @@ export class DatabaseUI extends DisposableObject {
handleChooseDatabaseFolder = async (
progress: ProgressCallback,
token: CancellationToken
): Promise<DatabaseItem | undefined> => {
): Promise<void> => {
try {
return await this.chooseAndSetDatabase(true, progress, token);
await this.chooseAndSetDatabase(true, progress, token);
} catch (e) {
void showAndLogErrorMessage(getErrorMessage(e));
return undefined;
@@ -458,9 +456,9 @@ export class DatabaseUI extends DisposableObject {
handleChooseDatabaseArchive = async (
progress: ProgressCallback,
token: CancellationToken
): Promise<DatabaseItem | undefined> => {
): Promise<void> => {
try {
return await this.chooseAndSetDatabase(false, progress, token);
await this.chooseAndSetDatabase(false, progress, token);
} catch (e) {
void showAndLogErrorMessage(getErrorMessage(e));
return undefined;
@@ -576,8 +574,7 @@ export class DatabaseUI extends DisposableObject {
// Search for upgrade scripts in any workspace folders available
await upgradeDatabaseExplicit(
this.queryServer,
await this.queryServer.upgradeDatabaseExplicit(
databaseItem,
progress,
token
@@ -592,8 +589,7 @@ export class DatabaseUI extends DisposableObject {
this.queryServer !== undefined &&
this.databaseManager.currentDatabaseItem !== undefined
) {
await clearCacheInDatabase(
this.queryServer,
await this.queryServer.clearCacheInDatabase(
this.databaseManager.currentDatabaseItem,
progress,
token

24
extensions/ql-vscode/src/databases.ts
View File

@@ -17,9 +17,8 @@ import {
import { zipArchiveScheme, encodeArchiveBasePath, decodeSourceArchiveUri, encodeSourceArchiveUri } from './archive-filesystem-provider';
import { DisposableObject } from './pure/disposable-object';
import { Logger, logger } from './logging';
import { registerDatabases, Dataset, deregisterDatabases } from './pure/messages';
import { QueryServerClient } from './queryserver-client';
import { getErrorMessage } from './pure/helpers-pure';
import { QueryRunner } from './queryRunner';
/**
* databases.ts
@@ -555,13 +554,13 @@ export class DatabaseManager extends DisposableObject {
constructor(
private readonly ctx: ExtensionContext,
private readonly qs: QueryServerClient,
private readonly qs: QueryRunner,
private readonly cli: cli.CodeQLCliServer,
public logger: Logger
) {
super();
qs.onDidStartQueryServer(this.reregisterDatabases.bind(this));
qs.onStart(this.reregisterDatabases.bind(this));
// Let this run async.
void this.loadPersistedState();
@@ -860,27 +859,14 @@ export class DatabaseManager extends DisposableObject {
token: vscode.CancellationToken,
dbItem: DatabaseItem,
) {
if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
const databases: Dataset[] = [{
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default'
}];
await this.qs.sendRequest(deregisterDatabases, { databases }, token, progress);
await this.qs.deregisterDatabase(progress, token, dbItem);
}
}
private async registerDatabase(
progress: ProgressCallback,
token: vscode.CancellationToken,
dbItem: DatabaseItem,
) {
if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
const databases: Dataset[] = [{
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default'
}];
await this.qs.sendRequest(registerDatabases, { databases }, token, progress);
}
await this.qs.registerDatabase(progress, token, dbItem);
}
private updatePersistedCurrentDatabaseItem(): void {

51
extensions/ql-vscode/src/extension.ts
View File

@@ -70,12 +70,11 @@ import { asError, assertNever, getErrorMessage } from './pure/helpers-pure';
import { spawnIdeServer } from './ide-server';
import { InterfaceManager } from './interface';
import { WebviewReveal } from './interface-utils';
import { ideServerLogger, logger, queryServerLogger } from './logging';
import { ideServerLogger, logger, ProgressReporter, queryServerLogger } from './logging';
import { QueryHistoryManager } from './query-history';
import { CompletedLocalQueryInfo, LocalQueryInfo } from './query-results';
import * as qsClient from './queryserver-client';
import * as qsClient from './legacy-query-server/queryserver-client';
import { displayQuickQuery } from './quick-query';
import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo } from './run-queries';
import { QLTestAdapterFactory } from './test-adapter';
import { TestUIService } from './test-ui';
import { CompareInterfaceManager } from './compare/compare-interface';
@@ -102,6 +101,9 @@ import { EvalLogViewer } from './eval-log-viewer';
import { SummaryLanguageSupport } from './log-insights/summary-language-support';
import { JoinOrderScannerProvider } from './log-insights/join-order';
import { LogScannerService } from './log-insights/log-scanner-service';
import { createInitialQueryInfo } from './run-queries-shared';
import { LegacyQueryRunner } from './legacy-query-server/legacyRunner';
import { QueryRunner } from './queryRunner';
/**
* extension.ts
@@ -164,7 +166,7 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
export interface CodeQLExtensionInterface {
readonly ctx: ExtensionContext;
readonly cliServer: CodeQLCliServer;
readonly qs: qsClient.QueryServerClient;
readonly qs: QueryRunner;
readonly distributionManager: DistributionManager;
readonly databaseManager: DatabaseManager;
readonly databaseUI: DatabaseUI;
@@ -416,21 +418,7 @@ async function activateWithInstalledDistribution(
ctx.subscriptions.push(statusBar);
void logger.log('Initializing query server client.');
const qs = new qsClient.QueryServerClient(
qlConfigurationListener,
cliServer,
{
logger: queryServerLogger,
contextStoragePath: getContextStoragePath(ctx),
},
(task) =>
Window.withProgress(
{ title: 'CodeQL query server', location: ProgressLocation.Window },
task
)
);
ctx.subscriptions.push(qs);
await qs.startQueryServer();
const qs = await createQueryServer(qlConfigurationListener, cliServer, ctx);
void logger.log('Initializing database manager.');
const dbm = new DatabaseManager(ctx, qs, cliServer, logger);
@@ -552,9 +540,7 @@ async function activateWithInstalledDistribution(
const item = new LocalQueryInfo(initialInfo, source);
qhm.addQuery(item);
try {
const completedQueryInfo = await compileAndRunQueryAgainstDatabase(
cliServer,
qs,
const completedQueryInfo = await qs.compileAndRunQueryAgainstDatabase(
databaseItem,
initialInfo,
queryStorageDir,
@@ -788,6 +774,7 @@ async function activateWithInstalledDistribution(
// requests for each query to run.
// Only do it if running multiple queries since this check is
// performed on each query run anyway.
// Don't do this with non destructive upgrades as the user won't see anything anyway.
await databaseUI.tryUpgradeCurrentDatabase(progress, token);
}
@@ -1139,6 +1126,26 @@ async function activateWithInstalledDistribution(
};
}
async function createQueryServer(qlConfigurationListener: QueryServerConfigListener, cliServer: CodeQLCliServer, ctx: ExtensionContext): Promise<QueryRunner> {
const qsOpts = {
logger: queryServerLogger,
contextStoragePath: getContextStoragePath(ctx),
};
const progressCallback = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable<void>) => Window.withProgress(
{ title: 'CodeQL query server', location: ProgressLocation.Window },
task
);
const qs = new qsClient.QueryServerClient(
qlConfigurationListener,
cliServer,
qsOpts,
progressCallback
);
ctx.subscriptions.push(qs);
await qs.startQueryServer();
return new LegacyQueryRunner(qs);
}
function getContextStoragePath(ctx: ExtensionContext) {
return ctx.storageUri?.fsPath || ctx.globalStorageUri.fsPath;
}

5
extensions/ql-vscode/src/interface.ts
View File

@@ -29,10 +29,9 @@ import {
RawResultsSortState,
} from './pure/interface-types';
import { Logger } from './logging';
import * as messages from './pure/messages';
import { commandRunner } from './commandRunner';
import { CompletedQueryInfo, interpretResultsSarif, interpretGraphResults } from './query-results';
import { QueryEvaluationInfo } from './run-queries';
import { QueryEvaluationInfo } from './run-queries-shared';
import { parseSarifLocation, parseSarifPlainTextMessage } from './pure/sarif-utils';
import {
WebviewReveal,
@@ -327,7 +326,7 @@ export class InterfaceManager extends AbstractInterfaceManager<IntoResultsViewMs
forceReveal: WebviewReveal,
shouldKeepOldResultsWhileRendering = false
): Promise<void> {
if (fullQuery.completedQuery.result.resultType !== messages.QueryResultType.SUCCESS) {
if (!fullQuery.completedQuery.sucessful) {
return;
}

30
extensions/ql-vscode/src/json-rpc-server.ts Normal file
View File

@@ -0,0 +1,30 @@
import { Logger } from './logging';
import * as cp from 'child_process';
import { Disposable } from 'vscode';
import { MessageConnection } from 'vscode-jsonrpc';
/** A running query server process and its associated message connection. */
export class ServerProcess implements Disposable {
child: cp.ChildProcess;
connection: MessageConnection;
logger: Logger;
constructor(child: cp.ChildProcess, connection: MessageConnection, logger: Logger) {
this.child = child;
this.connection = connection;
this.logger = logger;
}
dispose(): void {
void this.logger.log('Stopping query server...');
this.connection.dispose();
this.child.stdin!.end();
this.child.stderr!.destroy();
// TODO kill the process if it doesn't terminate after a certain time limit.
// On Windows, we usually have to terminate the process before closing its stdout.
this.child.stdout!.destroy();
void this.logger.log('Stopped query server.');
}
}

59
extensions/ql-vscode/src/legacy-query-server/legacyRunner.ts Normal file
View File

@@ -0,0 +1,59 @@
import { CancellationToken } from 'vscode';
import { ProgressCallback } from '../commandRunner';
import { DatabaseItem } from '../databases';
import { Dataset, deregisterDatabases, registerDatabases } from '../pure/legacy-messages';
import { InitialQueryInfo, LocalQueryInfo } from '../query-results';
import { QueryRunner } from '../queryRunner';
import { QueryWithResults } from '../run-queries-shared';
import { QueryServerClient } from './queryserver-client';
import { clearCacheInDatabase, compileAndRunQueryAgainstDatabase } from './run-queries';
import { upgradeDatabaseExplicit } from './upgrades';
export class LegacyQueryRunner extends QueryRunner {
constructor(public readonly qs: QueryServerClient) {
super();
}
get cliServer() {
return this.qs.cliServer;
}
async restartQueryServer(progress: ProgressCallback, token: CancellationToken): Promise<void> {
await this.qs.restartQueryServer(progress, token);
}
onStart(callBack: (progress: ProgressCallback, token: CancellationToken) => Promise<void>) {
this.qs.onDidStartQueryServer(callBack);
}
async clearCacheInDatabase(dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken): Promise<void> {
await clearCacheInDatabase(this.qs, dbItem, progress, token);
}
async compileAndRunQueryAgainstDatabase(dbItem: DatabaseItem, initialInfo: InitialQueryInfo, queryStorageDir: string, progress: ProgressCallback, token: CancellationToken, templates?: Record<string, string>, queryInfo?: LocalQueryInfo): Promise<QueryWithResults> {
return await compileAndRunQueryAgainstDatabase(this.qs.cliServer, this.qs, dbItem, initialInfo, queryStorageDir, progress, token, templates, queryInfo);
}
async deregisterDatabase(progress: ProgressCallback, token: CancellationToken, dbItem: DatabaseItem): Promise<void> {
if (dbItem.contents && (await this.qs.cliServer.cliConstraints.supportsDatabaseRegistration())) {
const databases: Dataset[] = [{
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default'
}];
await this.qs.sendRequest(deregisterDatabases, { databases }, token, progress);
}
}
async registerDatabase(progress: ProgressCallback, token: CancellationToken, dbItem: DatabaseItem): Promise<void> {
if (dbItem.contents && (await this.qs.cliServer.cliConstraints.supportsDatabaseRegistration())) {
const databases: Dataset[] = [{
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default'
}];
await this.qs.sendRequest(registerDatabases, { databases }, token, progress);
}
}
async upgradeDatabaseExplicit(dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken): Promise<void> {
await upgradeDatabaseExplicit(this.qs, dbItem, progress, token);
}
}

74
extensions/ql-vscode/src/queryserver-client.ts → extensions/ql-vscode/src/legacy-query-server/queryserver-client.ts
View File

@@ -1,49 +1,25 @@
import * as cp from 'child_process';
import * as path from 'path';
import * as fs from 'fs-extra';
import { DisposableObject } from './pure/disposable-object';
import { Disposable, CancellationToken, commands } from 'vscode';
import { createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc';
import * as cli from './cli';
import { QueryServerConfig } from './config';
import { Logger, ProgressReporter } from './logging';
import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from './pure/messages';
import * as messages from './pure/messages';
import { ProgressCallback, ProgressTask } from './commandRunner';
import { DisposableObject } from '../pure/disposable-object';
import { CancellationToken, commands } from 'vscode';
import { createMessageConnection, RequestType } from 'vscode-jsonrpc';
import * as cli from '../cli';
import { QueryServerConfig } from '../config';
import { Logger, ProgressReporter } from '../logging';
import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from '../pure/legacy-messages';
import * as messages from '../pure/legacy-messages';
import { ProgressCallback, ProgressTask } from '../commandRunner';
import { findQueryLogFile } from '../run-queries-shared';
import { ServerProcess } from '../json-rpc-server';
type WithProgressReporting = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable<void>) => Thenable<void>;
type ServerOpts = {
logger: Logger;
contextStoragePath: string;
}
/** A running query server process and its associated message connection. */
class ServerProcess implements Disposable {
child: cp.ChildProcess;
connection: MessageConnection;
logger: Logger;
constructor(child: cp.ChildProcess, connection: MessageConnection, logger: Logger) {
this.child = child;
this.connection = connection;
this.logger = logger;
}
dispose(): void {
void this.logger.log('Stopping query server...');
this.connection.dispose();
this.child.stdin!.end();
this.child.stderr!.destroy();
// TODO kill the process if it doesn't terminate after a certain time limit.
// On Windows, we usually have to terminate the process before closing its stdout.
this.child.stdout!.destroy();
void this.logger.log('Stopped query server.');
}
}
type WithProgressReporting = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable<void>) => Thenable<void>;
/**
* Client that manages a query server process.
* The server process is started upon initialization and tracked during its lifetime.
@@ -254,27 +230,3 @@ export class QueryServerClient extends DisposableObject {
}
}
}
export function findQueryLogFile(resultPath: string): string {
return path.join(resultPath, 'query.log');
}
export function findQueryEvalLogFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.jsonl');
}
export function findQueryEvalLogSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary');
}
export function findJsonQueryEvalLogSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary.jsonl');
}
export function findQueryEvalLogSummarySymbolsFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary.symbols.json');
}
export function findQueryEvalLogEndSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log-end.summary');
}

518
extensions/ql-vscode/src/legacy-query-server/run-queries.ts Normal file
View File

@@ -0,0 +1,518 @@
import * as crypto from 'crypto';
import * as fs from 'fs-extra';
import * as tmp from 'tmp-promise';
import * as path from 'path';
import {
CancellationToken,
Uri,
} from 'vscode';
import { ErrorCodes, ResponseError } from 'vscode-languageclient';
import * as cli from '../cli';
import { DatabaseItem, } from '../databases';
import {
getOnDiskWorkspaceFolders,
showAndLogErrorMessage,
showAndLogWarningMessage,
tryGetQueryMetadata,
upgradesTmpDir
} from '../helpers';
import { ProgressCallback } from '../commandRunner';
import { QueryMetadata } from '../pure/interface-types';
import { logger } from '../logging';
import * as messages from '../pure/legacy-messages';
import { InitialQueryInfo, LocalQueryInfo } from '../query-results';
import * as qsClient from './queryserver-client';
import { getErrorMessage } from '../pure/helpers-pure';
import { compileDatabaseUpgradeSequence, upgradeDatabaseExplicit } from './upgrades';
import { QueryEvaluationInfo, QueryWithResults } from '../run-queries-shared';
/**
* A collection of evaluation-time information about a query,
* including the query itself, and where we have decided to put
* temporary files associated with it, such as the compiled query
* output and results.
*/
export class QueryInProgress {
public queryEvalInfo: QueryEvaluationInfo;
/**
* Note that in the {@link FullQueryInfo.slurp} method, we create a QueryEvaluationInfo instance
* by explicitly setting the prototype in order to avoid calling this constructor.
*/
constructor(
readonly querySaveDir: string,
readonly dbItemPath: string,
databaseHasMetadataFile: boolean,
readonly queryDbscheme: string, // the dbscheme file the query expects, based on library path resolution
readonly quickEvalPosition?: messages.Position,
readonly metadata?: QueryMetadata,
readonly templates?: Record<string, string>,
) {
this.queryEvalInfo = new QueryEvaluationInfo(querySaveDir, dbItemPath, databaseHasMetadataFile, quickEvalPosition, metadata);
/**/
}
get compiledQueryPath() {
return path.join(this.querySaveDir, 'compiledQuery.qlo');
}
async run(
qs: qsClient.QueryServerClient,
upgradeQlo: string | undefined,
availableMlModels: cli.MlModelInfo[],
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
queryInfo?: LocalQueryInfo,
): Promise<messages.EvaluationResult> {
if (!dbItem.contents || dbItem.error) {
throw new Error('Can\'t run query on invalid database.');
}
let result: messages.EvaluationResult | null = null;
const callbackId = qs.registerCallback(res => {
result = {
...res,
logFileLocation: this.queryEvalInfo.logPath
};
});
const availableMlModelUris: messages.MlModel[] = availableMlModels.map(model => ({ uri: Uri.file(model.path).toString(true) }));
const queryToRun: messages.QueryToRun = {
resultsPath: this.queryEvalInfo.resultsPaths.resultsPath,
qlo: Uri.file(this.compiledQueryPath).toString(),
compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(),
allowUnknownTemplates: true,
templateValues: createSimpleTemplates(this.templates),
availableMlModels: availableMlModelUris,
id: callbackId,
timeoutSecs: qs.config.timeoutSecs,
};
const dataset: messages.Dataset = {
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default'
};
if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
await qs.sendRequest(messages.startLog, {
db: dataset,
logPath: this.queryEvalInfo.evalLogPath,
});
}
const params: messages.EvaluateQueriesParams = {
db: dataset,
evaluateId: callbackId,
queries: [queryToRun],
stopOnError: false,
useSequenceHint: false
};
try {
await qs.sendRequest(messages.runQueries, params, token, progress);
if (qs.config.customLogDirectory) {
void showAndLogWarningMessage(
`Custom log directories are no longer supported. The "codeQL.runningQueries.customLogDirectory" setting is deprecated. Unset the setting to stop seeing this message. Query logs saved to ${this.queryEvalInfo.logPath}.`
);
}
} finally {
qs.unRegisterCallback(callbackId);
if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
await qs.sendRequest(messages.endLog, {
db: dataset,
logPath: this.queryEvalInfo.evalLogPath,
});
if (await this.queryEvalInfo.hasEvalLog()) {
await this.queryEvalInfo.addQueryLogs(queryInfo, qs.cliServer, qs.logger);
} else {
void showAndLogWarningMessage(`Failed to write structured evaluator log to ${this.queryEvalInfo.evalLogPath}.`);
}
}
}
return result || {
evaluationTime: 0,
message: 'No result from server',
queryId: -1,
runId: callbackId,
resultType: messages.QueryResultType.OTHER_ERROR
};
}
async compile(
qs: qsClient.QueryServerClient,
program: messages.QlProgram,
progress: ProgressCallback,
token: CancellationToken,
): Promise<messages.CompilationMessage[]> {
let compiled: messages.CheckQueryResult | undefined;
try {
const target = this.quickEvalPosition ? {
quickEval: { quickEvalPos: this.quickEvalPosition }
} : { query: {} };
const params: messages.CompileQueryParams = {
compilationOptions: {
computeNoLocationUrls: true,
failOnWarnings: false,
fastCompilation: false,
includeDilInQlo: true,
localChecking: false,
noComputeGetUrl: false,
noComputeToString: false,
computeDefaultStrings: true,
emitDebugInfo: true
},
extraOptions: {
timeoutSecs: qs.config.timeoutSecs
},
queryToCheck: program,
resultPath: this.compiledQueryPath,
target,
};
compiled = await qs.sendRequest(messages.compileQuery, params, token, progress);
} finally {
void qs.logger.log(' - - - COMPILATION DONE - - - ', { additionalLogLocation: this.queryEvalInfo.logPath });
}
return (compiled?.messages || []).filter(msg => msg.severity === messages.Severity.ERROR);
}
}
export async function clearCacheInDatabase(
qs: qsClient.QueryServerClient,
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<messages.ClearCacheResult> {
if (dbItem.contents === undefined) {
throw new Error('Can\'t clear the cache in an invalid database.');
}
const db: messages.Dataset = {
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default',
};
const params: messages.ClearCacheParams = {
dryRun: false,
db,
};
return qs.sendRequest(messages.clearCache, params, token, progress);
}
/**
* Compare the dbscheme implied by the query `query` and that of the current database.
* - If they are compatible, do nothing.
* - If they are incompatible but the database can be upgraded, suggest that upgrade.
* - If they are incompatible and the database cannot be upgraded, throw an error.
*/
async function checkDbschemeCompatibility(
cliServer: cli.CodeQLCliServer,
qs: qsClient.QueryServerClient,
query: QueryInProgress,
qlProgram: messages.QlProgram,
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<void> {
const searchPath = getOnDiskWorkspaceFolders();
if (dbItem.contents?.dbSchemeUri !== undefined) {
const { finalDbscheme } = await cliServer.resolveUpgrades(dbItem.contents.dbSchemeUri.fsPath, searchPath, false);
const hash = async function(filename: string): Promise<string> {
return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex');
};
// At this point, we have learned about three dbschemes:
// the dbscheme of the actual database we're querying.
const dbschemeOfDb = await hash(dbItem.contents.dbSchemeUri.fsPath);
// the dbscheme of the query we're running, including the library we've resolved it to use.
const dbschemeOfLib = await hash(query.queryDbscheme);
// the database we're able to upgrade to
const upgradableTo = await hash(finalDbscheme);
if (upgradableTo != dbschemeOfLib) {
reportNoUpgradePath(qlProgram, query);
}
if (upgradableTo == dbschemeOfLib &&
dbschemeOfDb != dbschemeOfLib) {
// Try to upgrade the database
await upgradeDatabaseExplicit(
qs,
dbItem,
progress,
token
);
}
}
}
function reportNoUpgradePath(qlProgram: messages.QlProgram, query: QueryInProgress): void {
throw new Error(
`Query ${qlProgram.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.`
);
}
/**
* Compile a non-destructive upgrade.
*/
async function compileNonDestructiveUpgrade(
qs: qsClient.QueryServerClient,
upgradeTemp: tmp.DirectoryResult,
query: QueryInProgress,
qlProgram: messages.QlProgram,
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<string> {
if (!dbItem?.contents?.dbSchemeUri) {
throw new Error('Database is invalid, and cannot be upgraded.');
}
// When packaging is used, dependencies may exist outside of the workspace and they are always on the resolved search path.
// When packaging is not used, all dependencies are in the workspace.
const upgradesPath = (await qs.cliServer.cliConstraints.supportsPackaging())
? qlProgram.libraryPath
: getOnDiskWorkspaceFolders();
const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(
dbItem.contents.dbSchemeUri.fsPath,
upgradesPath,
true,
query.queryDbscheme
);
if (!matchesTarget) {
reportNoUpgradePath(qlProgram, query);
}
const result = await compileDatabaseUpgradeSequence(qs, dbItem, scripts, upgradeTemp, progress, token);
if (result.compiledUpgrade === undefined) {
const error = result.error || '[no error message available]';
throw new Error(error);
}
// We can upgrade to the actual target
qlProgram.dbschemePath = query.queryDbscheme;
// We are new enough that we will always support single file upgrades.
return result.compiledUpgrade;
}
export async function compileAndRunQueryAgainstDatabase(
cliServer: cli.CodeQLCliServer,
qs: qsClient.QueryServerClient,
dbItem: DatabaseItem,
initialInfo: InitialQueryInfo,
queryStorageDir: string,
progress: ProgressCallback,
token: CancellationToken,
templates?: Record<string, string>,
queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
): Promise<QueryWithResults> {
if (!dbItem.contents || !dbItem.contents.dbSchemeUri) {
throw new Error(`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`);
}
// Get the workspace folder paths.
const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
// Figure out the library path for the query.
const packConfig = await cliServer.resolveLibraryPath(diskWorkspaceFolders, initialInfo.queryPath);
if (!packConfig.dbscheme) {
throw new Error('Could not find a database scheme for this query. Please check that you have a valid qlpack.yml file for this query, which refers to a database scheme either in the `dbscheme` field or through one of its dependencies.');
}
// Check whether the query has an entirely different schema from the
// database. (Queries that merely need the database to be upgraded
// won't trigger this check)
// This test will produce confusing results if we ever change the name of the database schema files.
const querySchemaName = path.basename(packConfig.dbscheme);
const dbSchemaName = path.basename(dbItem.contents.dbSchemeUri.fsPath);
if (querySchemaName != dbSchemaName) {
void logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`);
throw new Error(`The query ${path.basename(initialInfo.queryPath)} cannot be run against the selected database (${dbItem.name}): their target languages are different. Please select a different database and try again.`);
}
const qlProgram: messages.QlProgram = {
// The project of the current document determines which library path
// we use. The `libraryPath` field in this server message is relative
// to the workspace root, not to the project root.
libraryPath: packConfig.libraryPath,
// Since we are compiling and running a query against a database,
// we use the database's DB scheme here instead of the DB scheme
// from the current document's project.
dbschemePath: dbItem.contents.dbSchemeUri.fsPath,
queryPath: initialInfo.queryPath
};
// Read the query metadata if possible, to use in the UI.
const metadata = await tryGetQueryMetadata(cliServer, qlProgram.queryPath);
let availableMlModels: cli.MlModelInfo[] = [];
if (!await cliServer.cliConstraints.supportsResolveMlModels()) {
void logger.log('Resolving ML models is unsupported by this version of the CLI. Running the query without any ML models.');
} else {
try {
availableMlModels = (await cliServer.resolveMlModels(diskWorkspaceFolders, initialInfo.queryPath)).models;
if (availableMlModels.length) {
void logger.log(`Found available ML models at the following paths: ${availableMlModels.map(x => `'${x.path}'`).join(', ')}.`);
} else {
void logger.log('Did not find any available ML models.');
}
} catch (e) {
const message = `Couldn't resolve available ML models for ${qlProgram.queryPath}. Running the ` +
`query without any ML models: ${e}.`;
void showAndLogErrorMessage(message);
}
}
const hasMetadataFile = (await dbItem.hasMetadataFile());
const query = new QueryInProgress(
path.join(queryStorageDir, initialInfo.id),
dbItem.databaseUri.fsPath,
hasMetadataFile,
packConfig.dbscheme,
initialInfo.quickEvalPosition,
metadata,
templates
);
await query.queryEvalInfo.createTimestampFile();
let upgradeDir: tmp.DirectoryResult | undefined;
try {
let upgradeQlo;
if (await cliServer.cliConstraints.supportsNonDestructiveUpgrades()) {
upgradeDir = await tmp.dir({ dir: upgradesTmpDir, unsafeCleanup: true });
upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, qlProgram, dbItem, progress, token);
} else {
await checkDbschemeCompatibility(cliServer, qs, query, qlProgram, dbItem, progress, token);
}
let errors;
try {
errors = await query.compile(qs, qlProgram, progress, token);
} catch (e) {
if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
return createSyntheticResult(query, 'Query cancelled');
} else {
throw e;
}
}
if (errors.length === 0) {
const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token, queryInfo);
if (result.resultType !== messages.QueryResultType.SUCCESS) {
const message = result.message || 'Failed to run query';
void logger.log(message);
void showAndLogErrorMessage(message);
}
const message = formatLegacyMessage(result);
return {
query: query.queryEvalInfo,
message: message,
sucessful: result.resultType == messages.QueryResultType.SUCCESS,
logFileLocation: result.logFileLocation,
dispose: () => {
qs.logger.removeAdditionalLogLocation(result.logFileLocation);
}
};
} else {
// Error dialogs are limited in size and scrollability,
// so we include a general description of the problem,
// and direct the user to the output window for the detailed compilation messages.
// However we don't show quick eval errors there so we need to display them anyway.
void qs.logger.log(
`Failed to compile query ${initialInfo.queryPath} against database scheme ${qlProgram.dbschemePath}:`,
{ additionalLogLocation: query.queryEvalInfo.logPath }
);
const formattedMessages: string[] = [];
for (const error of errors) {
const message = error.message || '[no error message available]';
const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
formattedMessages.push(formatted);
void qs.logger.log(formatted, { additionalLogLocation: query.queryEvalInfo.logPath });
}
if (initialInfo.isQuickEval && formattedMessages.length <= 2) {
// If there are more than 2 error messages, they will not be displayed well in a popup
// and will be trimmed by the function displaying the error popup. Accordingly, we only
// try to show the errors if there are 2 or less, otherwise we direct the user to the log.
void showAndLogErrorMessage('Quick evaluation compilation failed: ' + formattedMessages.join('\n'));
} else {
void showAndLogErrorMessage((initialInfo.isQuickEval ? 'Quick evaluation' : 'Query') + compilationFailedErrorTail);
}
return createSyntheticResult(query, 'Query had compilation errors');
}
} finally {
try {
await upgradeDir?.cleanup();
} catch (e) {
void qs.logger.log(
`Could not clean up the upgrades dir. Reason: ${getErrorMessage(e)}`,
{ additionalLogLocation: query.queryEvalInfo.logPath }
);
}
}
}
const compilationFailedErrorTail = ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
' and the query and database use the same target language. For more details on the error, go to View > Output,' +
' and choose CodeQL Query Server from the dropdown.';
export function formatLegacyMessage(result: messages.EvaluationResult) {
switch (result.resultType) {
case messages.QueryResultType.CANCELLATION:
return `cancelled after ${Math.round(result.evaluationTime / 1000)} seconds`;
case messages.QueryResultType.OOM:
return 'out of memory';
case messages.QueryResultType.SUCCESS:
return `finished in ${Math.round(result.evaluationTime / 1000)} seconds`;
case messages.QueryResultType.TIMEOUT:
return `timed out after ${Math.round(result.evaluationTime / 1000)} seconds`;
case messages.QueryResultType.OTHER_ERROR:
default:
return result.message ? `failed: ${result.message}` : 'failed';
}
}
/**
* Create a synthetic result for a query that failed to compile.
*/
function createSyntheticResult(
query: QueryInProgress,
message: string,
): QueryWithResults {
return {
query: query.queryEvalInfo,
message,
sucessful: false,
dispose: () => { /**/ },
};
}
function createSimpleTemplates(templates: Record<string, string> | undefined): messages.TemplateDefinitions | undefined {
if (!templates) {
return undefined;
}
const result: messages.TemplateDefinitions = {};
for (const key of Object.keys(templates)) {
result[key] = {
values: {
tuples: [[{ stringValue: templates[key] }]]
}
};
}
return result;
}

13
extensions/ql-vscode/src/upgrades.ts → extensions/ql-vscode/src/legacy-query-server/upgrades.ts
View File

@@ -1,12 +1,12 @@
import * as vscode from 'vscode';
import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, tmpDir } from './helpers';
import { ProgressCallback, UserCancellationException } from './commandRunner';
import { logger } from './logging';
import * as messages from './pure/messages';
import * as qsClient from './queryserver-client';
import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, tmpDir } from '../helpers';
import { ProgressCallback, UserCancellationException } from '../commandRunner';
import { logger } from '../logging';
import * as messages from '../pure/legacy-messages';
import * as qsClient from '../legacy-query-server/queryserver-client';
import * as tmp from 'tmp-promise';
import * as path from 'path';
import { DatabaseItem } from './databases';
import { DatabaseItem } from '../databases';
/**
* Maximum number of lines to include from database upgrade message,
@@ -16,7 +16,6 @@ import { DatabaseItem } from './databases';
const MAX_UPGRADE_MESSAGE_LINES = 10;
/**
* Compile a database upgrade sequence.
* Callers must check that this is valid with the current queryserver first.

97
extensions/ql-vscode/src/pure/messages.ts → extensions/ql-vscode/src/pure/legacy-messages.ts
View File

@@ -15,38 +15,7 @@
*/
import * as rpc from 'vscode-jsonrpc';
/**
* A position within a QL file.
*/
export interface Position {
/**
* The one-based index of the start line
*/
line: number;
/**
* The one-based offset of the start column within
* the start line in UTF-16 code-units
*/
column: number;
/**
* The one-based index of the end line line
*/
endLine: number;
/**
* The one-based offset of the end column within
* the end line in UTF-16 code-units
*/
endColumn: number;
/**
* The path of the file.
* If the file name is "Compiler Generated" the
* the position is not a real position but
* arises from compiler generated code.
*/
fileName: string;
}
import * as shared from './messages-shared';
/**
* A query that should be checked for any errors or warnings
@@ -258,28 +227,6 @@ export interface DILQuery {
dilSource: string;
}
/**
* The way of compiling the query, as a normal query
* or a subset of it. Note that precisely one of the two options should be set.
*/
export interface CompilationTarget {
/**
* Compile as a normal query
*/
query?: Record<string, never>;
/**
* Compile as a quick evaluation
*/
quickEval?: QuickEvalOptions;
}
/**
* Options for quick evaluation
*/
export interface QuickEvalOptions {
quickEvalPos?: Position;
}
/**
* The result of checking a query.
*/
@@ -1012,37 +959,20 @@ export type DeregisterDatabasesResult = {
};
/**
* Type for any action that could have progress messages.
* A position within a QL file.
*/
export interface WithProgressId<T> {
/**
* The main body
*/
body: T;
/**
* The id used to report progress updates
*/
progressId: number;
}
export type Position = shared.Position;
export interface ProgressMessage {
/**
* The id of the operation that is running
* The way of compiling the query, as a normal query
* or a subset of it. Note that precisely one of the two options should be set.
*/
id: number;
/**
* The current step
*/
step: number;
/**
* The maximum step. This *should* be constant for a single job.
*/
maxStep: number;
/**
* The current progress message
*/
message: string;
}
export type CompilationTarget = shared.CompilationTarget;
export type QuickEvalOptions = shared.QuickEvalOptions;
export type WithProgressId<T> = shared.WithProgressId<T>;
export type ProgressMessage = shared.ProgressMessage;
/**
* Check a Ql query for errors without compiling it
@@ -1120,7 +1050,4 @@ export const deregisterDatabases = new rpc.RequestType<
*/
export const completeQuery = new rpc.RequestType<EvaluationResult, Record<string, any>, void, void>('evaluation/queryCompleted');
/**
* A notification that the progress has been changed.
*/
export const progress = new rpc.NotificationType<ProgressMessage, void>('ql/progressUpdated');
export const progress = shared.progress;

110
extensions/ql-vscode/src/pure/messages-shared.ts Normal file
View File

@@ -0,0 +1,110 @@
/**
* Types for messages exchanged during jsonrpc communication with the
* the CodeQL query server.
*
* This file exists in the queryserver and in the vscode extension, and
* should be kept in sync between them.
*
* A note about the namespaces below, which look like they are
* essentially enums, namely Severity, ResultColumnKind, and
* QueryResultType. By design, for the sake of extensibility, clients
* receiving messages of this protocol are supposed to accept any
* number for any of these types. We commit to the given meaning of
* the numbers listed in constants in the namespaces, and we commit to
* the fact that any unknown QueryResultType value counts as an error.
*/
import * as rpc from 'vscode-jsonrpc';
/**
* A position within a QL file.
*/
export interface Position {
/**
* The one-based index of the start line
*/
line: number;
/**
* The one-based offset of the start column within
* the start line in UTF-16 code-units
*/
column: number;
/**
* The one-based index of the end line line
*/
endLine: number;
/**
* The one-based offset of the end column within
* the end line in UTF-16 code-units
*/
endColumn: number;
/**
* The path of the file.
* If the file name is "Compiler Generated" the
* the position is not a real position but
* arises from compiler generated code.
*/
fileName: string;
}
/**
* The way of compiling the query, as a normal query
* or a subset of it. Note that precisely one of the two options should be set.
*/
export interface CompilationTarget {
/**
* Compile as a normal query
*/
query?: Record<string, never>;
/**
* Compile as a quick evaluation
*/
quickEval?: QuickEvalOptions;
}
/**
* Options for quick evaluation
*/
export interface QuickEvalOptions {
quickEvalPos?: Position;
}
/**
* Type for any action that could have progress messages.
*/
export interface WithProgressId<T> {
/**
* The main body
*/
body: T;
/**
* The id used to report progress updates
*/
progressId: number;
}
export interface ProgressMessage {
/**
* The id of the operation that is running
*/
id: number;
/**
* The current step
*/
step: number;
/**
* The maximum step. This *should* be constant for a single job.
*/
maxStep: number;
/**
* The current progress message
*/
message: string;
}
/**
* A notification that the progress has been changed.
*/
export const progress = new rpc.NotificationType<ProgressMessage, void>('ql/progressUpdated');

18
extensions/ql-vscode/src/query-history.ts
View File

@@ -26,7 +26,6 @@ import {
} from './helpers';
import { logger } from './logging';
import { URLSearchParams } from 'url';
import { QueryServerClient } from './queryserver-client';
import { DisposableObject } from './pure/disposable-object';
import { commandRunner } from './commandRunner';
import { ONE_HOUR_IN_MS, TWO_HOURS_IN_MS } from './pure/time';
@@ -48,7 +47,8 @@ import { WebviewReveal } from './interface-utils';
import { EvalLogViewer } from './eval-log-viewer';
import EvalLogTreeBuilder from './eval-log-tree-builder';
import { EvalLogData, parseViewerData } from './pure/log-summary-parser';
import { QueryWithResults } from './run-queries';
import { QueryWithResults } from './run-queries-shared';
import { QueryRunner } from './queryRunner';
/**
* query-history.ts
@@ -329,7 +329,7 @@ export class QueryHistoryManager extends DisposableObject {
readonly onDidCompleteQuery = this._onDidCompleteQuery.event;
constructor(
private readonly qs: QueryServerClient,
private readonly qs: QueryRunner,
private readonly dbm: DatabaseManager,
private readonly localQueriesInterfaceManager: InterfaceManager,
private readonly remoteQueriesManager: RemoteQueriesManager,
@@ -791,7 +791,7 @@ export class QueryHistoryManager extends DisposableObject {
throw new Error('Please select a local query.');
}
if (!finalSingleItem.completedQuery?.didRunSuccessfully) {
if (!finalSingleItem.completedQuery?.sucessful) {
throw new Error('Please select a query that has completed successfully.');
}
@@ -1072,7 +1072,7 @@ export class QueryHistoryManager extends DisposableObject {
void this.tryOpenExternalFile(query.csvPath);
return;
}
if (await query.exportCsvResults(this.qs, query.csvPath)) {
if (await query.exportCsvResults(this.qs.cliServer, query.csvPath)) {
void this.tryOpenExternalFile(
query.csvPath
);
@@ -1091,7 +1091,7 @@ export class QueryHistoryManager extends DisposableObject {
}
await this.tryOpenExternalFile(
await finalSingleItem.completedQuery.query.ensureCsvAlerts(this.qs, this.dbm)
await finalSingleItem.completedQuery.query.ensureCsvAlerts(this.qs.cliServer, this.dbm)
);
}
@@ -1107,7 +1107,7 @@ export class QueryHistoryManager extends DisposableObject {
}
await this.tryOpenExternalFile(
await finalSingleItem.completedQuery.query.ensureDilPath(this.qs)
await finalSingleItem.completedQuery.query.ensureDilPath(this.qs.cliServer)
);
}
@@ -1232,7 +1232,7 @@ the file in the file explorer and dragging it into the workspace.`
if (!otherQuery.completedQuery) {
throw new Error('Please select a completed query.');
}
if (!otherQuery.completedQuery.didRunSuccessfully) {
if (!otherQuery.completedQuery.sucessful) {
throw new Error('Please select a successful query.');
}
if (otherQuery.initialInfo.databaseInfo.name !== dbName) {
@@ -1252,7 +1252,7 @@ the file in the file explorer and dragging it into the workspace.`
otherQuery !== singleItem &&
otherQuery.t === 'local' &&
otherQuery.completedQuery &&
otherQuery.completedQuery.didRunSuccessfully &&
otherQuery.completedQuery.sucessful &&
otherQuery.initialInfo.databaseInfo.name === dbName
)
.map((item) => ({

41
extensions/ql-vscode/src/query-results.ts
View File

@@ -1,7 +1,7 @@
import { CancellationTokenSource, env } from 'vscode';
import { QueryWithResults, QueryEvaluationInfo } from './run-queries';
import * as messages from './pure/messages';
import * as messages from './pure/messages-shared';
import * as legacyMessages from './pure/legacy-messages';
import * as cli from './cli';
import * as fs from 'fs-extra';
import * as path from 'path';
@@ -17,6 +17,8 @@ import {
import { DatabaseInfo } from './pure/interface-types';
import { QueryStatus } from './query-status';
import { RemoteQueryHistoryItem } from './remote-queries/remote-query-history-item';
import { QueryEvaluationInfo, QueryWithResults } from './run-queries-shared';
import { formatLegacyMessage } from './legacy-query-server/run-queries';
/**
* query-results.ts
@@ -44,7 +46,12 @@ export interface InitialQueryInfo {
export class CompletedQueryInfo implements QueryWithResults {
readonly query: QueryEvaluationInfo;
readonly result: messages.EvaluationResult;
readonly message?: string;
readonly sucessful?: boolean;
/**
* The legacy result. This is only set when loading from the query history.
*/
readonly result?: legacyMessages.EvaluationResult;
readonly logFileLocation?: string;
resultCount: number;
@@ -75,9 +82,9 @@ export class CompletedQueryInfo implements QueryWithResults {
evaluation: QueryWithResults,
) {
this.query = evaluation.query;
this.result = evaluation.result;
this.logFileLocation = evaluation.logFileLocation;
this.message = evaluation.message;
this.sucessful = evaluation.sucessful;
// Use the dispose method from the evaluation.
// The dispose will clean up any additional log locations that this
// query may have created.
@@ -92,18 +99,12 @@ export class CompletedQueryInfo implements QueryWithResults {
}
get statusString(): string {
switch (this.result.resultType) {
case messages.QueryResultType.CANCELLATION:
return `cancelled after ${Math.round(this.result.evaluationTime / 1000)} seconds`;
case messages.QueryResultType.OOM:
return 'out of memory';
case messages.QueryResultType.SUCCESS:
return `finished in ${Math.round(this.result.evaluationTime / 1000)} seconds`;
case messages.QueryResultType.TIMEOUT:
return `timed out after ${Math.round(this.result.evaluationTime / 1000)} seconds`;
case messages.QueryResultType.OTHER_ERROR:
default:
return this.result.message ? `failed: ${this.result.message}` : 'failed';
if (this.message) {
return this.message;
} else if (this.result) {
return formatLegacyMessage(this.result);
} else {
throw new Error('No status available');
}
}
@@ -115,10 +116,6 @@ export class CompletedQueryInfo implements QueryWithResults {
|| this.query.resultsPaths.resultsPath;
}
get didRunSuccessfully(): boolean {
return this.result.resultType === messages.QueryResultType.SUCCESS;
}
async updateSortState(
server: cli.CodeQLCliServer,
resultSetName: string,
@@ -302,7 +299,7 @@ export class LocalQueryInfo {
return QueryStatus.Failed;
} else if (!this.completedQuery) {
return QueryStatus.InProgress;
} else if (this.completedQuery.didRunSuccessfully) {
} else if (this.completedQuery.sucessful) {
return QueryStatus.Completed;
} else {
return QueryStatus.Failed;

2
extensions/ql-vscode/src/query-serialization.ts
View File

@@ -5,7 +5,7 @@ import { showAndLogErrorMessage } from './helpers';
import { asyncFilter, getErrorMessage, getErrorStack } from './pure/helpers-pure';
import { CompletedQueryInfo, LocalQueryInfo, QueryHistoryInfo } from './query-results';
import { QueryStatus } from './query-status';
import { QueryEvaluationInfo } from './run-queries';
import { QueryEvaluationInfo } from './run-queries-shared';
export async function slurpQueryHistory(fsPath: string): Promise<QueryHistoryInfo[]> {
try {

48
extensions/ql-vscode/src/queryRunner.ts Normal file
View File

@@ -0,0 +1,48 @@
import { CancellationToken } from 'vscode';
import { CodeQLCliServer } from './cli';
import { ProgressCallback } from './commandRunner';
import { DatabaseItem } from './databases';
import { InitialQueryInfo, LocalQueryInfo } from './query-results';
import { QueryWithResults } from './run-queries-shared';
export abstract class QueryRunner {
abstract restartQueryServer(progress: ProgressCallback, token: CancellationToken): Promise<void>;
abstract cliServer: CodeQLCliServer;
abstract onStart(arg0: (progress: ProgressCallback, token: CancellationToken) => Promise<void>): void;
abstract clearCacheInDatabase(
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken): Promise<void>;
abstract compileAndRunQueryAgainstDatabase(
dbItem: DatabaseItem,
initialInfo: InitialQueryInfo,
queryStorageDir: string,
progress: ProgressCallback,
token: CancellationToken,
templates?: Record<string, string>,
queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
): Promise<QueryWithResults>;
abstract deregisterDatabase(
progress: ProgressCallback,
token: CancellationToken,
dbItem: DatabaseItem,
): Promise<void>;
abstract registerDatabase(
progress: ProgressCallback,
token: CancellationToken,
dbItem: DatabaseItem,
): Promise<void>;
abstract upgradeDatabaseExplicit(
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<void>
}

585
extensions/ql-vscode/src/run-queries-shared.ts Normal file
View File

@@ -0,0 +1,585 @@
import * as messages from './pure/messages-shared';
import * as legacyMessages from './pure/legacy-messages';
import { DatabaseInfo, QueryMetadata } from './pure/interface-types';
import * as path from 'path';
import { createTimestampFile, showAndLogWarningMessage } from './helpers';
import {
ConfigurationTarget,
Range,
TextDocument,
TextEditor,
Uri,
window
} from 'vscode';
import * as config from './config';
import { UserCancellationException } from './commandRunner';
import * as fs from 'fs-extra';
import { ensureMetadataIsComplete, InitialQueryInfo, LocalQueryInfo } from './query-results';
import { isQuickQueryPath } from './quick-query';
import { nanoid } from 'nanoid';
import { CodeQLCliServer } from './cli';
import { SELECT_QUERY_NAME } from './contextual/locationFinder';
import { DatabaseManager } from './databases';
import { DecodedBqrsChunk } from './pure/bqrs-cli-types';
import { logger, Logger } from './logging';
import { generateSummarySymbolsFile } from './log-insights/summary-parser';
import { asError } from './pure/helpers-pure';
/**
* run-queries.ts
* --------------
*
* Compiling and running QL queries.
*/
export function findQueryLogFile(resultPath: string): string {
return path.join(resultPath, 'query.log');
}
function findQueryEvalLogFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.jsonl');
}
function findQueryEvalLogSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary');
}
function findJsonQueryEvalLogSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary.jsonl');
}
function findQueryEvalLogSummarySymbolsFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary.symbols.json');
}
function findQueryEvalLogEndSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log-end.summary');
}
export class QueryEvaluationInfo {
/**
* Note that in the {@link FullQueryInfo.slurp} method, we create a QueryEvaluationInfo instance
* by explicitly setting the prototype in order to avoid calling this constructor.
*/
constructor(
public readonly querySaveDir: string,
public readonly dbItemPath: string,
private readonly databaseHasMetadataFile: boolean,
public readonly quickEvalPosition?: messages.Position,
public readonly metadata?: QueryMetadata,
) {
/**/
}
get dilPath() {
return path.join(this.querySaveDir, 'results.dil');
}
/**
* Get the path that the compiled query is if it exists. Note that it only exists when using the legacy query server.
*/
get compileQueryPath() {
return path.join(this.querySaveDir, 'compiledQuery.qlo');
}
get csvPath() {
return path.join(this.querySaveDir, 'results.csv');
}
get logPath() {
return findQueryLogFile(this.querySaveDir);
}
get evalLogPath() {
return findQueryEvalLogFile(this.querySaveDir);
}
get evalLogSummaryPath() {
return findQueryEvalLogSummaryFile(this.querySaveDir);
}
get jsonEvalLogSummaryPath() {
return findJsonQueryEvalLogSummaryFile(this.querySaveDir);
}
get evalLogSummarySymbolsPath() {
return findQueryEvalLogSummarySymbolsFile(this.querySaveDir);
}
get evalLogEndSummaryPath() {
return findQueryEvalLogEndSummaryFile(this.querySaveDir);
}
get resultsPaths() {
return {
resultsPath: path.join(this.querySaveDir, 'results.bqrs'),
interpretedResultsPath: path.join(this.querySaveDir,
this.metadata?.kind === 'graph'
? 'graphResults'
: 'interpretedResults.sarif'
),
};
}
getSortedResultSetPath(resultSetName: string) {
return path.join(this.querySaveDir, `sortedResults-${resultSetName}.bqrs`);
}
/**
* Creates a file in the query directory that indicates when this query was created.
* This is important for keeping track of when queries should be removed.
*/
async createTimestampFile() {
await createTimestampFile(this.querySaveDir);
}
/**
* Holds if this query can in principle produce interpreted results.
*/
canHaveInterpretedResults(): boolean {
if (!this.databaseHasMetadataFile) {
void logger.log('Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.');
return false;
}
const kind = this.metadata?.kind;
const hasKind = !!kind;
if (!hasKind) {
void logger.log('Cannot produce interpreted results since the query does not have @kind metadata.');
return false;
}
// Graph queries only return interpreted results if we are in canary mode.
if (kind === 'graph') {
return config.isCanary();
}
// table is the default query kind. It does not produce interpreted results.
// any query kind that is not table can, in principle, produce interpreted results.
return kind !== 'table';
}
/**
* Holds if this query actually has produced interpreted results.
*/
async hasInterpretedResults(): Promise<boolean> {
return fs.pathExists(this.resultsPaths.interpretedResultsPath);
}
/**
* Holds if this query already has DIL produced
*/
async hasDil(): Promise<boolean> {
return fs.pathExists(this.dilPath);
}
/**
* Holds if this query already has CSV results produced
*/
async hasCsv(): Promise<boolean> {
return fs.pathExists(this.csvPath);
}
/**
* Returns the path to the DIL file produced by this query. If the query has not yet produced DIL,
* this will return first create the DIL file and then return the path to the DIL file.
*/
async ensureDilPath(cliServer: CodeQLCliServer): Promise<string> {
if (await this.hasDil()) {
return this.dilPath;
}
const compiledQuery = path.join(this.querySaveDir, 'compiledQuery.qlo');
if (!(await fs.pathExists(compiledQuery))) {
throw new Error(
`Cannot create DIL because compiled query is missing. ${compiledQuery}`
);
}
await cliServer.generateDil(compiledQuery, this.dilPath);
return this.dilPath;
}
/**
* Holds if this query already has a completed structured evaluator log
*/
async hasEvalLog(): Promise<boolean> {
return fs.pathExists(this.evalLogPath);
}
async addQueryLogs(queryInfo: LocalQueryInfo, cliServer: CodeQLCliServer, logger: Logger) {
queryInfo.evalLogLocation = this.evalLogPath;
queryInfo.evalLogSummaryLocation = await this.generateHumanReadableLogSummary(cliServer);
void this.logEndSummary(queryInfo.evalLogSummaryLocation, logger); // Logged asynchrnously
if (config.isCanary()) { // Generate JSON summary for viewer.
await cliServer.generateJsonLogSummary(this.evalLogPath, this.jsonEvalLogSummaryPath);
queryInfo.jsonEvalLogSummaryLocation = this.jsonEvalLogSummaryPath;
await generateSummarySymbolsFile(this.evalLogSummaryPath, this.evalLogSummarySymbolsPath);
queryInfo.evalLogSummarySymbolsLocation = this.evalLogSummarySymbolsPath;
}
}
/**
* Calls the appropriate CLI command to generate a human-readable log summary.
* @param qs The query server client.
* @returns The path to the log summary, or `undefined` if the summary could not be generated. */
private async generateHumanReadableLogSummary(cliServer: CodeQLCliServer): Promise<string | undefined> {
try {
await cliServer.generateLogSummary(this.evalLogPath, this.evalLogSummaryPath, this.evalLogEndSummaryPath);
return this.evalLogSummaryPath;
} catch (e) {
const err = asError(e);
void showAndLogWarningMessage(`Failed to generate human-readable structured evaluator log summary. Reason: ${err.message}`);
return undefined;
}
}
/**
* Logs the end summary to the Output window and log file.
* @param logSummaryPath Path to the human-readable log summary
* @param qs The query server client.
*/
private async logEndSummary(logSummaryPath: string | undefined, logger: Logger): Promise<void> {
if (logSummaryPath === undefined) {
// Failed to generate the log, so we don't expect an end summary either.
return;
}
try {
const endSummaryContent = await fs.readFile(this.evalLogEndSummaryPath, 'utf-8');
void logger.log(' --- Evaluator Log Summary --- ', { additionalLogLocation: this.logPath });
void logger.log(endSummaryContent, { additionalLogLocation: this.logPath });
} catch (e) {
void showAndLogWarningMessage(`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`);
}
}
/**
* Creates the CSV file containing the results of this query. This will only be called if the query
* does not have interpreted results and the CSV file does not already exist.
*
* @return Promise<true> if the operation creates the file. Promise<false> if the operation does
* not create the file.
*
* @throws Error if the operation fails.
*/
async exportCsvResults(cliServer: CodeQLCliServer, csvPath: string): Promise<boolean> {
const resultSet = await this.chooseResultSet(cliServer);
if (!resultSet) {
void showAndLogWarningMessage('Query has no result set.');
return false;
}
let stopDecoding = false;
const out = fs.createWriteStream(csvPath);
const promise: Promise<boolean> = new Promise((resolve, reject) => {
out.on('finish', () => resolve(true));
out.on('error', () => {
if (!stopDecoding) {
stopDecoding = true;
reject(new Error(`Failed to write CSV results to ${csvPath}`));
}
});
});
let nextOffset: number | undefined = 0;
do {
const chunk: DecodedBqrsChunk = await cliServer.bqrsDecode(this.resultsPaths.resultsPath, resultSet, {
pageSize: 100,
offset: nextOffset,
});
chunk.tuples.forEach((tuple) => {
out.write(tuple.map((v, i) =>
chunk.columns[i].kind === 'String'
? `"${typeof v === 'string' ? v.replaceAll('"', '""') : v}"`
: v
).join(',') + '\n');
});
nextOffset = chunk.next;
} while (nextOffset && !stopDecoding);
out.end();
return promise;
}
/**
* Choose the name of the result set to run. If the `#select` set exists, use that. Otherwise,
* arbitrarily choose the first set. Most of the time, this will be correct.
*
* If the query has no result sets, then return undefined.
*/
async chooseResultSet(cliServer: CodeQLCliServer) {
const resultSets = (await cliServer.bqrsInfo(this.resultsPaths.resultsPath, 0))['result-sets'];
if (!resultSets.length) {
return undefined;
}
if (resultSets.find(r => r.name === SELECT_QUERY_NAME)) {
return SELECT_QUERY_NAME;
}
return resultSets[0].name;
}
/**
* Returns the path to the CSV alerts interpretation of this query results. If CSV results have
* not yet been produced, this will return first create the CSV results and then return the path.
*
* This method only works for queries with interpreted results.
*/
async ensureCsvAlerts(cliServer: CodeQLCliServer, dbm: DatabaseManager): Promise<string> {
if (await this.hasCsv()) {
return this.csvPath;
}
const dbItem = dbm.findDatabaseItem(Uri.file(this.dbItemPath));
if (!dbItem) {
throw new Error(`Cannot produce CSV results because database is missing. ${this.dbItemPath}`);
}
let sourceInfo;
if (dbItem.sourceArchive !== undefined) {
sourceInfo = {
sourceArchive: dbItem.sourceArchive.fsPath,
sourceLocationPrefix: await dbItem.getSourceLocationPrefix(
cliServer
),
};
}
await cliServer.generateResultsCsv(ensureMetadataIsComplete(this.metadata), this.resultsPaths.resultsPath, this.csvPath, sourceInfo);
return this.csvPath;
}
/**
* Cleans this query's results directory.
*/
async deleteQuery(): Promise<void> {
await fs.remove(this.querySaveDir);
}
}
export interface QueryWithResults {
readonly query: QueryEvaluationInfo;
readonly logFileLocation?: string;
readonly dispose: () => void;
readonly sucessful?: boolean;
readonly message?: string;
readonly result?: legacyMessages.EvaluationResult
}
/**
* Information about which query will be to be run. `quickEvalPosition` and `quickEvalText`
* is only filled in if the query is a quick query.
*/
interface SelectedQuery {
queryPath: string;
quickEvalPosition?: messages.Position;
quickEvalText?: string;
}
/**
* Determines which QL file to run during an invocation of `Run Query` or `Quick Evaluation`, as follows:
* - If the command was called by clicking on a file, then use that file.
* - Otherwise, use the file open in the current editor.
* - In either case, prompt the user to save the file if it is open with unsaved changes.
* - For `Quick Evaluation`, ensure the selected file is also the one open in the editor,
* and use the selected region.
* @param selectedResourceUri The selected resource when the command was run.
* @param quickEval Whether the command being run is `Quick Evaluation`.
*/
export async function determineSelectedQuery(
selectedResourceUri: Uri | undefined,
quickEval: boolean,
range?: Range
): Promise<SelectedQuery> {
const editor = window.activeTextEditor;
// Choose which QL file to use.
let queryUri: Uri;
if (selectedResourceUri) {
// A resource was passed to the command handler, so use it.
queryUri = selectedResourceUri;
} else {
// No resource was passed to the command handler, so obtain it from the active editor.
// This usually happens when the command is called from the Command Palette.
if (editor === undefined) {
throw new Error('No query was selected. Please select a query and try again.');
} else {
queryUri = editor.document.uri;
}
}
if (queryUri.scheme !== 'file') {
throw new Error('Can only run queries that are on disk.');
}
const queryPath = queryUri.fsPath;
if (quickEval) {
if (!(queryPath.endsWith('.ql') || queryPath.endsWith('.qll'))) {
throw new Error('The selected resource is not a CodeQL file; It should have the extension ".ql" or ".qll".');
}
} else {
if (!(queryPath.endsWith('.ql'))) {
throw new Error('The selected resource is not a CodeQL query file; It should have the extension ".ql".');
}
}
// Whether we chose the file from the active editor or from a context menu,
// if the same file is open with unsaved changes in the active editor,
// then prompt the user to save it first.
if (editor !== undefined && editor.document.uri.fsPath === queryPath) {
if (await promptUserToSaveChanges(editor.document)) {
await editor.document.save();
}
}
let quickEvalPosition: messages.Position | undefined = undefined;
let quickEvalText: string | undefined = undefined;
if (quickEval) {
if (editor == undefined) {
throw new Error('Can\'t run quick evaluation without an active editor.');
}
if (editor.document.fileName !== queryPath) {
// For Quick Evaluation we expect these to be the same.
// Report an error if we end up in this (hopefully unlikely) situation.
throw new Error('The selected resource for quick evaluation should match the active editor.');
}
quickEvalPosition = await getSelectedPosition(editor, range);
if (!editor.selection?.isEmpty) {
quickEvalText = editor.document.getText(editor.selection);
} else {
// capture the entire line if the user didn't select anything
const line = editor.document.lineAt(editor.selection.active.line);
quickEvalText = line.text.trim();
}
}
return { queryPath, quickEvalPosition, quickEvalText };
}
/** Gets the selected position within the given editor. */
async function getSelectedPosition(editor: TextEditor, range?: Range): Promise<messages.Position> {
const selectedRange = range || editor.selection;
const pos = selectedRange.start;
const posEnd = selectedRange.end;
// Convert from 0-based to 1-based line and column numbers.
return {
fileName: await convertToQlPath(editor.document.fileName),
line: pos.line + 1,
column: pos.character + 1,
endLine: posEnd.line + 1,
endColumn: posEnd.character + 1
};
}
/**
* Prompts the user to save `document` if it has unsaved changes.
*
* @param document The document to save.
*
* @returns true if we should save changes and false if we should continue without saving changes.
* @throws UserCancellationException if we should abort whatever operation triggered this prompt
*/
async function promptUserToSaveChanges(document: TextDocument): Promise<boolean> {
if (document.isDirty) {
if (config.AUTOSAVE_SETTING.getValue()) {
return true;
}
else {
const yesItem = { title: 'Yes', isCloseAffordance: false };
const alwaysItem = { title: 'Always Save', isCloseAffordance: false };
const noItem = { title: 'No (run version on disk)', isCloseAffordance: false };
const cancelItem = { title: 'Cancel', isCloseAffordance: true };
const message = 'Query file has unsaved changes. Save now?';
const chosenItem = await window.showInformationMessage(
message,
{ modal: true },
yesItem, alwaysItem, noItem, cancelItem
);
if (chosenItem === alwaysItem) {
await config.AUTOSAVE_SETTING.updateValue(true, ConfigurationTarget.Workspace);
return true;
}
if (chosenItem === yesItem) {
return true;
}
if (chosenItem === cancelItem) {
throw new UserCancellationException('Query run cancelled.', true);
}
}
}
return false;
}
/**
* @param filePath This needs to be equivalent to Java's `Path.toRealPath(NO_FOLLOW_LINKS)`
*/
async function convertToQlPath(filePath: string): Promise<string> {
if (process.platform === 'win32') {
if (path.parse(filePath).root === filePath) {
// Java assumes uppercase drive letters are canonical.
return filePath.toUpperCase();
} else {
const dir = await convertToQlPath(path.dirname(filePath));
const fileName = path.basename(filePath);
const fileNames = await fs.readdir(dir);
for (const name of fileNames) {
// Leave the locale argument empty so that the default OS locale is used.
// We do this because this operation works on filesystem entities, which
// use the os locale, regardless of the locale of the running VS Code instance.
if (fileName.localeCompare(name, undefined, { sensitivity: 'accent' }) === 0) {
return path.join(dir, name);
}
}
}
throw new Error('Can\'t convert path to form suitable for QL:' + filePath);
} else {
return filePath;
}
}
/**
* Determines the initial information for a query. This is everything of interest
* we know about this query that is available before it is run.
*
* @param selectedQueryUri The Uri of the document containing the query to be run.
* @param databaseInfo The database to run the query against.
* @param isQuickEval true if this is a quick evaluation.
* @param range the selection range of the query to be run. Only used if isQuickEval is true.
* @returns The initial information for the query to be run.
*/
export async function createInitialQueryInfo(
selectedQueryUri: Uri | undefined,
databaseInfo: DatabaseInfo,
isQuickEval: boolean,
range?: Range
): Promise<InitialQueryInfo> {
// Determine which query to run, based on the selection and the active editor.
const { queryPath, quickEvalPosition, quickEvalText } = await determineSelectedQuery(selectedQueryUri, isQuickEval, range);
return {
queryPath,
isQuickEval,
isQuickQuery: isQuickQueryPath(queryPath),
databaseInfo,
id: `${path.basename(queryPath)}-${nanoid()}`,
start: new Date(),
... (isQuickEval ? {
queryText: quickEvalText!, // if this query is quick eval, it must have quick eval text
quickEvalPosition: quickEvalPosition
} : {
queryText: await fs.readFile(queryPath, 'utf8')
})
};
}

997
extensions/ql-vscode/src/run-queries.ts
View File

@@ -1,997 +0,0 @@
import * as crypto from 'crypto';
import * as fs from 'fs-extra';
import * as tmp from 'tmp-promise';
import * as path from 'path';
import { nanoid } from 'nanoid';
import {
CancellationToken,
ConfigurationTarget,
Range,
TextDocument,
TextEditor,
Uri,
window
} from 'vscode';
import { ErrorCodes, ResponseError } from 'vscode-languageclient';
import * as cli from './cli';
import * as config from './config';
import { DatabaseItem, DatabaseManager } from './databases';
import {
createTimestampFile,
getOnDiskWorkspaceFolders,
showAndLogErrorMessage,
showAndLogWarningMessage,
tryGetQueryMetadata,
upgradesTmpDir
} from './helpers';
import { ProgressCallback, UserCancellationException } from './commandRunner';
import { DatabaseInfo, QueryMetadata } from './pure/interface-types';
import { logger } from './logging';
import * as messages from './pure/messages';
import { InitialQueryInfo, LocalQueryInfo } from './query-results';
import * as qsClient from './queryserver-client';
import { isQuickQueryPath } from './quick-query';
import { compileDatabaseUpgradeSequence, upgradeDatabaseExplicit } from './upgrades';
import { ensureMetadataIsComplete } from './query-results';
import { SELECT_QUERY_NAME } from './contextual/locationFinder';
import { DecodedBqrsChunk } from './pure/bqrs-cli-types';
import { asError, getErrorMessage } from './pure/helpers-pure';
import { generateSummarySymbolsFile } from './log-insights/summary-parser';
/**
* run-queries.ts
* --------------
*
* Compiling and running QL queries.
*/
/**
* Information about which query will be to be run. `quickEvalPosition` and `quickEvalText`
* is only filled in if the query is a quick query.
*/
interface SelectedQuery {
queryPath: string;
quickEvalPosition?: messages.Position;
quickEvalText?: string;
}
/**
* A collection of evaluation-time information about a query,
* including the query itself, and where we have decided to put
* temporary files associated with it, such as the compiled query
* output and results.
*/
export class QueryEvaluationInfo {
/**
* Note that in the {@link FullQueryInfo.slurp} method, we create a QueryEvaluationInfo instance
* by explicitly setting the prototype in order to avoid calling this constructor.
*/
constructor(
public readonly querySaveDir: string,
public readonly dbItemPath: string,
private readonly databaseHasMetadataFile: boolean,
public readonly queryDbscheme: string, // the dbscheme file the query expects, based on library path resolution
public readonly quickEvalPosition?: messages.Position,
public readonly metadata?: QueryMetadata,
public readonly templates?: messages.TemplateDefinitions
) {
/**/
}
get dilPath() {
return path.join(this.querySaveDir, 'results.dil');
}
get csvPath() {
return path.join(this.querySaveDir, 'results.csv');
}
get compiledQueryPath() {
return path.join(this.querySaveDir, 'compiledQuery.qlo');
}
get logPath() {
return qsClient.findQueryLogFile(this.querySaveDir);
}
get evalLogPath() {
return qsClient.findQueryEvalLogFile(this.querySaveDir);
}
get evalLogSummaryPath() {
return qsClient.findQueryEvalLogSummaryFile(this.querySaveDir);
}
get jsonEvalLogSummaryPath() {
return qsClient.findJsonQueryEvalLogSummaryFile(this.querySaveDir);
}
get evalLogSummarySymbolsPath() {
return qsClient.findQueryEvalLogSummarySymbolsFile(this.querySaveDir);
}
get evalLogEndSummaryPath() {
return qsClient.findQueryEvalLogEndSummaryFile(this.querySaveDir);
}
get resultsPaths() {
return {
resultsPath: path.join(this.querySaveDir, 'results.bqrs'),
interpretedResultsPath: path.join(this.querySaveDir,
this.metadata?.kind === 'graph'
? 'graphResults'
: 'interpretedResults.sarif'
),
};
}
getSortedResultSetPath(resultSetName: string) {
return path.join(this.querySaveDir, `sortedResults-${resultSetName}.bqrs`);
}
/**
* Creates a file in the query directory that indicates when this query was created.
* This is important for keeping track of when queries should be removed.
*/
async createTimestampFile() {
await createTimestampFile(this.querySaveDir);
}
async run(
qs: qsClient.QueryServerClient,
upgradeQlo: string | undefined,
availableMlModels: cli.MlModelInfo[],
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
queryInfo?: LocalQueryInfo,
): Promise<messages.EvaluationResult> {
if (!dbItem.contents || dbItem.error) {
throw new Error('Can\'t run query on invalid database.');
}
let result: messages.EvaluationResult | null = null;
const callbackId = qs.registerCallback(res => {
result = {
...res,
logFileLocation: this.logPath
};
});
const availableMlModelUris: messages.MlModel[] = availableMlModels.map(model => ({ uri: Uri.file(model.path).toString(true) }));
const queryToRun: messages.QueryToRun = {
resultsPath: this.resultsPaths.resultsPath,
qlo: Uri.file(this.compiledQueryPath).toString(),
compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(),
allowUnknownTemplates: true,
templateValues: this.templates,
availableMlModels: availableMlModelUris,
id: callbackId,
timeoutSecs: qs.config.timeoutSecs,
};
const dataset: messages.Dataset = {
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default'
};
if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
await qs.sendRequest(messages.startLog, {
db: dataset,
logPath: this.evalLogPath,
});
}
const params: messages.EvaluateQueriesParams = {
db: dataset,
evaluateId: callbackId,
queries: [queryToRun],
stopOnError: false,
useSequenceHint: false
};
try {
await qs.sendRequest(messages.runQueries, params, token, progress);
if (qs.config.customLogDirectory) {
void showAndLogWarningMessage(
`Custom log directories are no longer supported. The "codeQL.runningQueries.customLogDirectory" setting is deprecated. Unset the setting to stop seeing this message. Query logs saved to ${this.logPath}.`
);
}
} finally {
qs.unRegisterCallback(callbackId);
if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
await qs.sendRequest(messages.endLog, {
db: dataset,
logPath: this.evalLogPath,
});
if (await this.hasEvalLog()) {
queryInfo.evalLogLocation = this.evalLogPath;
queryInfo.evalLogSummaryLocation = await this.generateHumanReadableLogSummary(qs);
void this.logEndSummary(queryInfo.evalLogSummaryLocation, qs); // Logged asynchrnously
if (config.isCanary()) { // Generate JSON summary for viewer.
await qs.cliServer.generateJsonLogSummary(this.evalLogPath, this.jsonEvalLogSummaryPath);
queryInfo.jsonEvalLogSummaryLocation = this.jsonEvalLogSummaryPath;
await generateSummarySymbolsFile(this.evalLogSummaryPath, this.evalLogSummarySymbolsPath);
queryInfo.evalLogSummarySymbolsLocation = this.evalLogSummarySymbolsPath;
}
} else {
void showAndLogWarningMessage(`Failed to write structured evaluator log to ${this.evalLogPath}.`);
}
}
}
return result || {
evaluationTime: 0,
message: 'No result from server',
queryId: -1,
runId: callbackId,
resultType: messages.QueryResultType.OTHER_ERROR
};
}
async compile(
qs: qsClient.QueryServerClient,
program: messages.QlProgram,
progress: ProgressCallback,
token: CancellationToken,
): Promise<messages.CompilationMessage[]> {
let compiled: messages.CheckQueryResult | undefined;
try {
const target = this.quickEvalPosition ? {
quickEval: { quickEvalPos: this.quickEvalPosition }
} : { query: {} };
const params: messages.CompileQueryParams = {
compilationOptions: {
computeNoLocationUrls: true,
failOnWarnings: false,
fastCompilation: false,
includeDilInQlo: true,
localChecking: false,
noComputeGetUrl: false,
noComputeToString: false,
computeDefaultStrings: true,
emitDebugInfo: true
},
extraOptions: {
timeoutSecs: qs.config.timeoutSecs
},
queryToCheck: program,
resultPath: this.compiledQueryPath,
target,
};
compiled = await qs.sendRequest(messages.compileQuery, params, token, progress);
} finally {
void qs.logger.log(' - - - COMPILATION DONE - - - ', { additionalLogLocation: this.logPath });
}
return (compiled?.messages || []).filter(msg => msg.severity === messages.Severity.ERROR);
}
/**
* Holds if this query can in principle produce interpreted results.
*/
canHaveInterpretedResults(): boolean {
if (!this.databaseHasMetadataFile) {
void logger.log('Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.');
return false;
}
const kind = this.metadata?.kind;
const hasKind = !!kind;
if (!hasKind) {
void logger.log('Cannot produce interpreted results since the query does not have @kind metadata.');
return false;
}
// Graph queries only return interpreted results if we are in canary mode.
if (kind === 'graph') {
return config.isCanary();
}
// table is the default query kind. It does not produce interpreted results.
// any query kind that is not table can, in principle, produce interpreted results.
return kind !== 'table';
}
/**
* Holds if this query actually has produced interpreted results.
*/
async hasInterpretedResults(): Promise<boolean> {
return fs.pathExists(this.resultsPaths.interpretedResultsPath);
}
/**
* Holds if this query already has DIL produced
*/
async hasDil(): Promise<boolean> {
return fs.pathExists(this.dilPath);
}
/**
* Holds if this query already has CSV results produced
*/
async hasCsv(): Promise<boolean> {
return fs.pathExists(this.csvPath);
}
/**
* Returns the path to the DIL file produced by this query. If the query has not yet produced DIL,
* this will return first create the DIL file and then return the path to the DIL file.
*/
async ensureDilPath(qs: qsClient.QueryServerClient): Promise<string> {
if (await this.hasDil()) {
return this.dilPath;
}
if (!(await fs.pathExists(this.compiledQueryPath))) {
throw new Error(
`Cannot create DIL because compiled query is missing. ${this.compiledQueryPath}`
);
}
await qs.cliServer.generateDil(this.compiledQueryPath, this.dilPath);
return this.dilPath;
}
/**
* Holds if this query already has a completed structured evaluator log
*/
async hasEvalLog(): Promise<boolean> {
return fs.pathExists(this.evalLogPath);
}
/**
* Calls the appropriate CLI command to generate a human-readable log summary.
* @param qs The query server client.
* @returns The path to the log summary, or `undefined` if the summary could not be generated.
*/
private async generateHumanReadableLogSummary(qs: qsClient.QueryServerClient): Promise<string | undefined> {
try {
await qs.cliServer.generateLogSummary(this.evalLogPath, this.evalLogSummaryPath, this.evalLogEndSummaryPath);
return this.evalLogSummaryPath;
} catch (e) {
const err = asError(e);
void showAndLogWarningMessage(`Failed to generate human-readable structured evaluator log summary. Reason: ${err.message}`);
return undefined;
}
}
/**
* Logs the end summary to the Output window and log file.
* @param logSummaryPath Path to the human-readable log summary
* @param qs The query server client.
*/
private async logEndSummary(logSummaryPath: string | undefined, qs: qsClient.QueryServerClient): Promise<void> {
if (logSummaryPath === undefined) {
// Failed to generate the log, so we don't expect an end summary either.
return;
}
try {
const endSummaryContent = await fs.readFile(this.evalLogEndSummaryPath, 'utf-8');
void qs.logger.log(' --- Evaluator Log Summary --- ', { additionalLogLocation: this.logPath });
void qs.logger.log(endSummaryContent, { additionalLogLocation: this.logPath });
} catch (e) {
void showAndLogWarningMessage(`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`);
}
}
/**
* Creates the CSV file containing the results of this query. This will only be called if the query
* does not have interpreted results and the CSV file does not already exist.
*
* @return Promise<true> if the operation creates the file. Promise<false> if the operation does
* not create the file.
*
* @throws Error if the operation fails.
*/
async exportCsvResults(qs: qsClient.QueryServerClient, csvPath: string): Promise<boolean> {
const resultSet = await this.chooseResultSet(qs);
if (!resultSet) {
void showAndLogWarningMessage('Query has no result set.');
return false;
}
let stopDecoding = false;
const out = fs.createWriteStream(csvPath);
const promise: Promise<boolean> = new Promise((resolve, reject) => {
out.on('finish', () => resolve(true));
out.on('error', () => {
if (!stopDecoding) {
stopDecoding = true;
reject(new Error(`Failed to write CSV results to ${csvPath}`));
}
});
});
let nextOffset: number | undefined = 0;
do {
const chunk: DecodedBqrsChunk = await qs.cliServer.bqrsDecode(this.resultsPaths.resultsPath, resultSet, {
pageSize: 100,
offset: nextOffset,
});
chunk.tuples.forEach((tuple) => {
out.write(tuple.map((v, i) =>
chunk.columns[i].kind === 'String'
? `"${typeof v === 'string' ? v.replaceAll('"', '""') : v}"`
: v
).join(',') + '\n');
});
nextOffset = chunk.next;
} while (nextOffset && !stopDecoding);
out.end();
return promise;
}
/**
* Choose the name of the result set to run. If the `#select` set exists, use that. Otherwise,
* arbitrarily choose the first set. Most of the time, this will be correct.
*
* If the query has no result sets, then return undefined.
*/
async chooseResultSet(qs: qsClient.QueryServerClient) {
const resultSets = (await qs.cliServer.bqrsInfo(this.resultsPaths.resultsPath, 0))['result-sets'];
if (!resultSets.length) {
return undefined;
}
if (resultSets.find(r => r.name === SELECT_QUERY_NAME)) {
return SELECT_QUERY_NAME;
}
return resultSets[0].name;
}
/**
* Returns the path to the CSV alerts interpretation of this query results. If CSV results have
* not yet been produced, this will return first create the CSV results and then return the path.
*
* This method only works for queries with interpreted results.
*/
async ensureCsvAlerts(qs: qsClient.QueryServerClient, dbm: DatabaseManager): Promise<string> {
if (await this.hasCsv()) {
return this.csvPath;
}
const dbItem = dbm.findDatabaseItem(Uri.file(this.dbItemPath));
if (!dbItem) {
throw new Error(`Cannot produce CSV results because database is missing. ${this.dbItemPath}`);
}
let sourceInfo;
if (dbItem.sourceArchive !== undefined) {
sourceInfo = {
sourceArchive: dbItem.sourceArchive.fsPath,
sourceLocationPrefix: await dbItem.getSourceLocationPrefix(
qs.cliServer
),
};
}
await qs.cliServer.generateResultsCsv(ensureMetadataIsComplete(this.metadata), this.resultsPaths.resultsPath, this.csvPath, sourceInfo);
return this.csvPath;
}
/**
* Cleans this query's results directory.
*/
async deleteQuery(): Promise<void> {
await fs.remove(this.querySaveDir);
}
}
export interface QueryWithResults {
readonly query: QueryEvaluationInfo;
readonly result: messages.EvaluationResult;
readonly logFileLocation?: string;
readonly dispose: () => void;
}
export async function clearCacheInDatabase(
qs: qsClient.QueryServerClient,
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<messages.ClearCacheResult> {
if (dbItem.contents === undefined) {
throw new Error('Can\'t clear the cache in an invalid database.');
}
const db: messages.Dataset = {
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default',
};
const params: messages.ClearCacheParams = {
dryRun: false,
db,
};
return qs.sendRequest(messages.clearCache, params, token, progress);
}
/**
* @param filePath This needs to be equivalent to Java's `Path.toRealPath(NO_FOLLOW_LINKS)`
*/
async function convertToQlPath(filePath: string): Promise<string> {
if (process.platform === 'win32') {
if (path.parse(filePath).root === filePath) {
// Java assumes uppercase drive letters are canonical.
return filePath.toUpperCase();
} else {
const dir = await convertToQlPath(path.dirname(filePath));
const fileName = path.basename(filePath);
const fileNames = await fs.readdir(dir);
for (const name of fileNames) {
// Leave the locale argument empty so that the default OS locale is used.
// We do this because this operation works on filesystem entities, which
// use the os locale, regardless of the locale of the running VS Code instance.
if (fileName.localeCompare(name, undefined, { sensitivity: 'accent' }) === 0) {
return path.join(dir, name);
}
}
}
throw new Error('Can\'t convert path to form suitable for QL:' + filePath);
} else {
return filePath;
}
}
/** Gets the selected position within the given editor. */
async function getSelectedPosition(editor: TextEditor, range?: Range): Promise<messages.Position> {
const selectedRange = range || editor.selection;
const pos = selectedRange.start;
const posEnd = selectedRange.end;
// Convert from 0-based to 1-based line and column numbers.
return {
fileName: await convertToQlPath(editor.document.fileName),
line: pos.line + 1,
column: pos.character + 1,
endLine: posEnd.line + 1,
endColumn: posEnd.character + 1
};
}
/**
* Compare the dbscheme implied by the query `query` and that of the current database.
* - If they are compatible, do nothing.
* - If they are incompatible but the database can be upgraded, suggest that upgrade.
* - If they are incompatible and the database cannot be upgraded, throw an error.
*/
async function checkDbschemeCompatibility(
cliServer: cli.CodeQLCliServer,
qs: qsClient.QueryServerClient,
query: QueryEvaluationInfo,
qlProgram: messages.QlProgram,
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<void> {
const searchPath = getOnDiskWorkspaceFolders();
if (dbItem.contents?.dbSchemeUri !== undefined) {
const { finalDbscheme } = await cliServer.resolveUpgrades(dbItem.contents.dbSchemeUri.fsPath, searchPath, false);
const hash = async function(filename: string): Promise<string> {
return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex');
};
// At this point, we have learned about three dbschemes:
// the dbscheme of the actual database we're querying.
const dbschemeOfDb = await hash(dbItem.contents.dbSchemeUri.fsPath);
// the dbscheme of the query we're running, including the library we've resolved it to use.
const dbschemeOfLib = await hash(query.queryDbscheme);
// the database we're able to upgrade to
const upgradableTo = await hash(finalDbscheme);
if (upgradableTo != dbschemeOfLib) {
reportNoUpgradePath(qlProgram, query);
}
if (upgradableTo == dbschemeOfLib &&
dbschemeOfDb != dbschemeOfLib) {
// Try to upgrade the database
await upgradeDatabaseExplicit(
qs,
dbItem,
progress,
token
);
}
}
}
function reportNoUpgradePath(qlProgram: messages.QlProgram, query: QueryEvaluationInfo): void {
throw new Error(
`Query ${qlProgram.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.`
);
}
/**
* Compile a non-destructive upgrade.
*/
async function compileNonDestructiveUpgrade(
qs: qsClient.QueryServerClient,
upgradeTemp: tmp.DirectoryResult,
query: QueryEvaluationInfo,
qlProgram: messages.QlProgram,
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<string> {
if (!dbItem?.contents?.dbSchemeUri) {
throw new Error('Database is invalid, and cannot be upgraded.');
}
// When packaging is used, dependencies may exist outside of the workspace and they are always on the resolved search path.
// When packaging is not used, all dependencies are in the workspace.
const upgradesPath = (await qs.cliServer.cliConstraints.supportsPackaging())
? qlProgram.libraryPath
: getOnDiskWorkspaceFolders();
const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(
dbItem.contents.dbSchemeUri.fsPath,
upgradesPath,
true,
query.queryDbscheme
);
if (!matchesTarget) {
reportNoUpgradePath(qlProgram, query);
}
const result = await compileDatabaseUpgradeSequence(qs, dbItem, scripts, upgradeTemp, progress, token);
if (result.compiledUpgrade === undefined) {
const error = result.error || '[no error message available]';
throw new Error(error);
}
// We can upgrade to the actual target
qlProgram.dbschemePath = query.queryDbscheme;
// We are new enough that we will always support single file upgrades.
return result.compiledUpgrade;
}
/**
* Prompts the user to save `document` if it has unsaved changes.
*
* @param document The document to save.
*
* @returns true if we should save changes and false if we should continue without saving changes.
* @throws UserCancellationException if we should abort whatever operation triggered this prompt
*/
async function promptUserToSaveChanges(document: TextDocument): Promise<boolean> {
if (document.isDirty) {
if (config.AUTOSAVE_SETTING.getValue()) {
return true;
}
else {
const yesItem = { title: 'Yes', isCloseAffordance: false };
const alwaysItem = { title: 'Always Save', isCloseAffordance: false };
const noItem = { title: 'No (run version on disk)', isCloseAffordance: false };
const cancelItem = { title: 'Cancel', isCloseAffordance: true };
const message = 'Query file has unsaved changes. Save now?';
const chosenItem = await window.showInformationMessage(
message,
{ modal: true },
yesItem, alwaysItem, noItem, cancelItem
);
if (chosenItem === alwaysItem) {
await config.AUTOSAVE_SETTING.updateValue(true, ConfigurationTarget.Workspace);
return true;
}
if (chosenItem === yesItem) {
return true;
}
if (chosenItem === cancelItem) {
throw new UserCancellationException('Query run cancelled.', true);
}
}
}
return false;
}
/**
* Determines which QL file to run during an invocation of `Run Query` or `Quick Evaluation`, as follows:
* - If the command was called by clicking on a file, then use that file.
* - Otherwise, use the file open in the current editor.
* - In either case, prompt the user to save the file if it is open with unsaved changes.
* - For `Quick Evaluation`, ensure the selected file is also the one open in the editor,
* and use the selected region.
* @param selectedResourceUri The selected resource when the command was run.
* @param quickEval Whether the command being run is `Quick Evaluation`.
*/
export async function determineSelectedQuery(
selectedResourceUri: Uri | undefined,
quickEval: boolean,
range?: Range
): Promise<SelectedQuery> {
const editor = window.activeTextEditor;
// Choose which QL file to use.
let queryUri: Uri;
if (selectedResourceUri) {
// A resource was passed to the command handler, so use it.
queryUri = selectedResourceUri;
} else {
// No resource was passed to the command handler, so obtain it from the active editor.
// This usually happens when the command is called from the Command Palette.
if (editor === undefined) {
throw new Error('No query was selected. Please select a query and try again.');
} else {
queryUri = editor.document.uri;
}
}
if (queryUri.scheme !== 'file') {
throw new Error('Can only run queries that are on disk.');
}
const queryPath = queryUri.fsPath;
if (quickEval) {
if (!(queryPath.endsWith('.ql') || queryPath.endsWith('.qll'))) {
throw new Error('The selected resource is not a CodeQL file; It should have the extension ".ql" or ".qll".');
}
} else {
if (!(queryPath.endsWith('.ql'))) {
throw new Error('The selected resource is not a CodeQL query file; It should have the extension ".ql".');
}
}
// Whether we chose the file from the active editor or from a context menu,
// if the same file is open with unsaved changes in the active editor,
// then prompt the user to save it first.
if (editor !== undefined && editor.document.uri.fsPath === queryPath) {
if (await promptUserToSaveChanges(editor.document)) {
await editor.document.save();
}
}
let quickEvalPosition: messages.Position | undefined = undefined;
let quickEvalText: string | undefined = undefined;
if (quickEval) {
if (editor == undefined) {
throw new Error('Can\'t run quick evaluation without an active editor.');
}
if (editor.document.fileName !== queryPath) {
// For Quick Evaluation we expect these to be the same.
// Report an error if we end up in this (hopefully unlikely) situation.
throw new Error('The selected resource for quick evaluation should match the active editor.');
}
quickEvalPosition = await getSelectedPosition(editor, range);
if (!editor.selection?.isEmpty) {
quickEvalText = editor.document.getText(editor.selection);
} else {
// capture the entire line if the user didn't select anything
const line = editor.document.lineAt(editor.selection.active.line);
quickEvalText = line.text.trim();
}
}
return { queryPath, quickEvalPosition, quickEvalText };
}
export async function compileAndRunQueryAgainstDatabase(
cliServer: cli.CodeQLCliServer,
qs: qsClient.QueryServerClient,
dbItem: DatabaseItem,
initialInfo: InitialQueryInfo,
queryStorageDir: string,
progress: ProgressCallback,
token: CancellationToken,
templates?: messages.TemplateDefinitions,
queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
): Promise<QueryWithResults> {
if (!dbItem.contents || !dbItem.contents.dbSchemeUri) {
throw new Error(`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`);
}
// Get the workspace folder paths.
const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
// Figure out the library path for the query.
const packConfig = await cliServer.resolveLibraryPath(diskWorkspaceFolders, initialInfo.queryPath);
if (!packConfig.dbscheme) {
throw new Error('Could not find a database scheme for this query. Please check that you have a valid qlpack.yml file for this query, which refers to a database scheme either in the `dbscheme` field or through one of its dependencies.');
}
// Check whether the query has an entirely different schema from the
// database. (Queries that merely need the database to be upgraded
// won't trigger this check)
// This test will produce confusing results if we ever change the name of the database schema files.
const querySchemaName = path.basename(packConfig.dbscheme);
const dbSchemaName = path.basename(dbItem.contents.dbSchemeUri.fsPath);
if (querySchemaName != dbSchemaName) {
void logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`);
throw new Error(`The query ${path.basename(initialInfo.queryPath)} cannot be run against the selected database (${dbItem.name}): their target languages are different. Please select a different database and try again.`);
}
const qlProgram: messages.QlProgram = {
// The project of the current document determines which library path
// we use. The `libraryPath` field in this server message is relative
// to the workspace root, not to the project root.
libraryPath: packConfig.libraryPath,
// Since we are compiling and running a query against a database,
// we use the database's DB scheme here instead of the DB scheme
// from the current document's project.
dbschemePath: dbItem.contents.dbSchemeUri.fsPath,
queryPath: initialInfo.queryPath
};
// Read the query metadata if possible, to use in the UI.
const metadata = await tryGetQueryMetadata(cliServer, qlProgram.queryPath);
let availableMlModels: cli.MlModelInfo[] = [];
if (!await cliServer.cliConstraints.supportsResolveMlModels()) {
void logger.log('Resolving ML models is unsupported by this version of the CLI. Running the query without any ML models.');
} else {
try {
availableMlModels = (await cliServer.resolveMlModels(diskWorkspaceFolders, initialInfo.queryPath)).models;
if (availableMlModels.length) {
void logger.log(`Found available ML models at the following paths: ${availableMlModels.map(x => `'${x.path}'`).join(', ')}.`);
} else {
void logger.log('Did not find any available ML models.');
}
} catch (e) {
const message = `Couldn't resolve available ML models for ${qlProgram.queryPath}. Running the ` +
`query without any ML models: ${e}.`;
void showAndLogErrorMessage(message);
}
}
const hasMetadataFile = (await dbItem.hasMetadataFile());
const query = new QueryEvaluationInfo(
path.join(queryStorageDir, initialInfo.id),
dbItem.databaseUri.fsPath,
hasMetadataFile,
packConfig.dbscheme,
initialInfo.quickEvalPosition,
metadata,
templates
);
await query.createTimestampFile();
let upgradeDir: tmp.DirectoryResult | undefined;
try {
let upgradeQlo;
if (await qs.cliServer.cliConstraints.supportsNonDestructiveUpgrades()) {
upgradeDir = await tmp.dir({ dir: upgradesTmpDir, unsafeCleanup: true });
upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, qlProgram, dbItem, progress, token);
} else {
await checkDbschemeCompatibility(cliServer, qs, query, qlProgram, dbItem, progress, token);
}
let errors;
try {
errors = await query.compile(qs, qlProgram, progress, token);
} catch (e) {
if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
return createSyntheticResult(query, 'Query cancelled', messages.QueryResultType.CANCELLATION);
} else {
throw e;
}
}
if (errors.length === 0) {
const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token, queryInfo);
if (result.resultType !== messages.QueryResultType.SUCCESS) {
const message = result.message || 'Failed to run query';
void logger.log(message);
void showAndLogErrorMessage(message);
}
return {
query,
result,
logFileLocation: result.logFileLocation,
dispose: () => {
qs.logger.removeAdditionalLogLocation(result.logFileLocation);
}
};
} else {
// Error dialogs are limited in size and scrollability,
// so we include a general description of the problem,
// and direct the user to the output window for the detailed compilation messages.
// However we don't show quick eval errors there so we need to display them anyway.
void qs.logger.log(
`Failed to compile query ${initialInfo.queryPath} against database scheme ${qlProgram.dbschemePath}:`,
{ additionalLogLocation: query.logPath }
);
const formattedMessages: string[] = [];
for (const error of errors) {
const message = error.message || '[no error message available]';
const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
formattedMessages.push(formatted);
void qs.logger.log(formatted, { additionalLogLocation: query.logPath });
}
if (initialInfo.isQuickEval && formattedMessages.length <= 2) {
// If there are more than 2 error messages, they will not be displayed well in a popup
// and will be trimmed by the function displaying the error popup. Accordingly, we only
// try to show the errors if there are 2 or less, otherwise we direct the user to the log.
void showAndLogErrorMessage('Quick evaluation compilation failed: ' + formattedMessages.join('\n'));
} else {
void showAndLogErrorMessage((initialInfo.isQuickEval ? 'Quick evaluation' : 'Query') + compilationFailedErrorTail);
}
return createSyntheticResult(query, 'Query had compilation errors', messages.QueryResultType.OTHER_ERROR);
}
} finally {
try {
await upgradeDir?.cleanup();
} catch (e) {
void qs.logger.log(
`Could not clean up the upgrades dir. Reason: ${getErrorMessage(e)}`,
{ additionalLogLocation: query.logPath }
);
}
}
}
/**
* Determines the initial information for a query. This is everything of interest
* we know about this query that is available before it is run.
*
* @param selectedQueryUri The Uri of the document containing the query to be run.
* @param databaseInfo The database to run the query against.
* @param isQuickEval true if this is a quick evaluation.
* @param range the selection range of the query to be run. Only used if isQuickEval is true.
* @returns The initial information for the query to be run.
*/
export async function createInitialQueryInfo(
selectedQueryUri: Uri | undefined,
databaseInfo: DatabaseInfo,
isQuickEval: boolean,
range?: Range
): Promise<InitialQueryInfo> {
// Determine which query to run, based on the selection and the active editor.
const { queryPath, quickEvalPosition, quickEvalText } = await determineSelectedQuery(selectedQueryUri, isQuickEval, range);
return {
queryPath,
isQuickEval,
isQuickQuery: isQuickQueryPath(queryPath),
databaseInfo,
id: `${path.basename(queryPath)}-${nanoid()}`,
start: new Date(),
... (isQuickEval ? {
queryText: quickEvalText!, // if this query is quick eval, it must have quick eval text
quickEvalPosition: quickEvalPosition
} : {
queryText: await fs.readFile(queryPath, 'utf8')
})
};
}
const compilationFailedErrorTail = ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
' and the query and database use the same target language. For more details on the error, go to View > Output,' +
' and choose CodeQL Query Server from the dropdown.';
/**
* Create a synthetic result for a query that failed to compile.
*/
function createSyntheticResult(
query: QueryEvaluationInfo,
message: string,
resultType: number
): QueryWithResults {
return {
query,
result: {
evaluationTime: 0,
resultType: resultType,
queryId: -1,
runId: -1,
message
},
dispose: () => { /**/ },
};
}

34
extensions/ql-vscode/src/vscode-tests/cli-integration/query.test.ts → extensions/ql-vscode/src/vscode-tests/cli-integration/legacy-query.test.ts
View File

@@ -4,14 +4,16 @@ import * as path from 'path';
import * as tmp from 'tmp';
import * as url from 'url';
import { CancellationTokenSource } from 'vscode-jsonrpc';
import * as messages from '../../pure/messages';
import * as qsClient from '../../queryserver-client';
import * as messages from '../../pure/legacy-messages';
import * as qsClient from '../../legacy-query-server/queryserver-client';
import * as cli from '../../cli';
import { CellValue } from '../../pure/bqrs-cli-types';
import { extensions } from 'vscode';
import { CodeQLExtensionInterface } from '../../extension';
import { fail } from 'assert';
import { skipIfNoCodeQL } from '../ensureCli';
import { QueryServerClient } from '../../legacy-query-server/queryserver-client';
import { logger, ProgressReporter } from '../../logging';
const baseDir = path.join(__dirname, '../../../test/data');
@@ -88,7 +90,7 @@ const db: messages.Dataset = {
workingSet: 'default',
};
describe('using the query server', function() {
describe('using the legacy query server', function() {
before(function() {
skipIfNoCodeQL(this);
});
@@ -97,28 +99,37 @@ describe('using the query server', function() {
// ensure they are all written with standard anonymous functions.
this.timeout(20000);
const nullProgressReporter: ProgressReporter = { report: () => { /** ignore */ } };
let qs: qsClient.QueryServerClient;
let cliServer: cli.CodeQLCliServer;
const queryServerStarted = new Checkpoint<void>();
beforeEach(async () => {
before(async () => {
try {
const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
if ('cliServer' in extension && 'qs' in extension) {
if ('cliServer' in extension) {
cliServer = extension.cliServer;
qs = extension.qs;
cliServer.quiet = true;
qs = new QueryServerClient({
codeQlPath: (await extension.distributionManager.getCodeQlPathWithoutVersionCheck()) || '',
debug: false,
cacheSize: 0,
numThreads: 1,
saveCache: false,
timeoutSecs: 0
}, cliServer, {
contextStoragePath: tmpDir.name,
logger
}, task => task(nullProgressReporter, new CancellationTokenSource().token));
await qs.startQueryServer();
} else {
throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
}
} catch (e) {
fail(e as Error);
}
});
it('should be able to start the query server', async function() {
await qs.startQueryServer();
await queryServerStarted.resolve();
});
for (const queryTestCase of queryTestCases) {
@@ -134,7 +145,6 @@ describe('using the query server', function() {
});
it(`should be able to compile query ${queryName}`, async function() {
await queryServerStarted.done();
expect(fs.existsSync(queryTestCase.queryPath)).to.be.true;
try {
const qlProgram: messages.QlProgram = {

21
extensions/ql-vscode/src/vscode-tests/cli-integration/queries.test.ts
View File

@@ -10,12 +10,11 @@ import { DatabaseItem, DatabaseManager } from '../../databases';
import { CodeQLExtensionInterface } from '../../extension';
import { cleanDatabases, dbLoc, storagePath } from './global.helper';
import { importArchiveDatabase } from '../../databaseFetcher';
import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo } from '../../run-queries';
import { CodeQLCliServer } from '../../cli';
import { QueryServerClient } from '../../queryserver-client';
import { skipIfNoCodeQL } from '../ensureCli';
import { QueryResultType } from '../../pure/messages';
import { tmpDir } from '../../helpers';
import { createInitialQueryInfo } from '../../run-queries-shared';
import { QueryRunner } from '../../queryRunner';
/**
@@ -31,7 +30,7 @@ describe('Queries', function() {
let dbItem: DatabaseItem;
let databaseManager: DatabaseManager;
let cli: CodeQLCliServer;
let qs: QueryServerClient;
let qs: QueryRunner;
let sandbox: sinon.SinonSandbox;
let progress: sinon.SinonSpy;
let token: CancellationToken;
@@ -104,9 +103,7 @@ describe('Queries', function() {
it('should run a query', async () => {
try {
const queryPath = path.join(__dirname, 'data', 'simple-query.ql');
const result = await compileAndRunQueryAgainstDatabase(
cli,
qs,
const result = qs.compileAndRunQueryAgainstDatabase(
dbItem,
await mockInitialQueryInfo(queryPath),
path.join(tmpDir.name, 'mock-storage-path'),
@@ -115,7 +112,7 @@ describe('Queries', function() {
);
// just check that the query was successful
expect(result.result.resultType).to.eq(QueryResultType.SUCCESS);
expect((await result).sucessful).to.eq(true);
} catch (e) {
console.error('Test Failed');
fail(e as Error);
@@ -127,9 +124,7 @@ describe('Queries', function() {
try {
await commands.executeCommand('codeQL.restartQueryServer');
const queryPath = path.join(__dirname, 'data', 'simple-query.ql');
const result = await compileAndRunQueryAgainstDatabase(
cli,
qs,
const result = await qs.compileAndRunQueryAgainstDatabase(
dbItem,
await mockInitialQueryInfo(queryPath),
path.join(tmpDir.name, 'mock-storage-path'),
@@ -137,9 +132,7 @@ describe('Queries', function() {
token
);
// this message would indicate that the databases were not properly reregistered
expect(result.result.message).not.to.eq('No result from server');
expect(result.result.resultType).to.eq(QueryResultType.SUCCESS);
expect(result.sucessful).to.eq(true);
} catch (e) {
console.error('Test Failed');
fail(e as Error);

50
extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts
View File

@@ -14,12 +14,11 @@ import {
findSourceArchive
} from '../../databases';
import { Logger } from '../../logging';
import { QueryServerClient } from '../../queryserver-client';
import { registerDatabases } from '../../pure/messages';
import { ProgressCallback } from '../../commandRunner';
import { CodeQLCliServer } from '../../cli';
import { encodeArchiveBasePath, encodeSourceArchiveUri } from '../../archive-filesystem-provider';
import { testDisposeHandler } from '../test-dispose-handler';
import { QueryRunner } from '../../queryRunner';
describe('databases', () => {
@@ -33,7 +32,8 @@ describe('databases', () => {
let updateSpy: sinon.SinonSpy;
let getSpy: sinon.SinonStub;
let dbChangedHandler: sinon.SinonSpy;
let sendRequestSpy: sinon.SinonSpy;
let registerSpy: sinon.SinonSpy;
let deregisterSpy: sinon.SinonSpy;
let supportsDatabaseRegistrationSpy: sinon.SinonStub;
let supportsLanguageNameSpy: sinon.SinonStub;
let resolveDatabaseSpy: sinon.SinonStub;
@@ -48,7 +48,8 @@ describe('databases', () => {
updateSpy = sandbox.spy();
getSpy = sandbox.stub();
getSpy.returns([]);
sendRequestSpy = sandbox.stub();
registerSpy = sandbox.stub();
deregisterSpy = sandbox.stub();
dbChangedHandler = sandbox.spy();
supportsDatabaseRegistrationSpy = sandbox.stub();
supportsDatabaseRegistrationSpy.resolves(true);
@@ -65,9 +66,10 @@ describe('databases', () => {
storagePath: dir.name
} as unknown as ExtensionContext,
{
sendRequest: sendRequestSpy,
onDidStartQueryServer: () => { /**/ }
} as unknown as QueryServerClient,
registerDatabase: registerSpy,
deregisterDatabase: deregisterSpy,
onStart: () => { /**/ }
} as unknown as QueryRunner,
{
cliConstraints: {
supportsLanguageName: supportsLanguageNameSpy,
@@ -259,12 +261,6 @@ describe('databases', () => {
// similar test as above, but also check the call to sendRequestSpy to make sure they send the
// registration messages.
const mockDbItem = createMockDB();
const registration = {
databases: [{
dbDir: mockDbItem.contents!.datasetUri.fsPath,
workingSet: 'default'
}]
};
sandbox.stub(fs, 'remove').resolves();
@@ -274,7 +270,7 @@ describe('databases', () => {
mockDbItem
);
// Should have registered this database
expect(sendRequestSpy).to.have.been.calledWith(registerDatabases, registration, {}, {});
expect(registerSpy).to.have.been.calledWith({}, {}, mockDbItem);
await databaseManager.removeDatabaseItem(
{} as ProgressCallback,
@@ -283,31 +279,7 @@ describe('databases', () => {
);
// Should have deregistered this database
expect(sendRequestSpy).to.have.been.calledWith(registerDatabases, registration, {}, {});
});
it('should avoid registration when query server does not support it', async () => {
// similar test as above, but now pretend query server doesn't support registration
supportsDatabaseRegistrationSpy.resolves(false);
const mockDbItem = createMockDB();
sandbox.stub(fs, 'remove').resolves();
await (databaseManager as any).addDatabaseItem(
{} as ProgressCallback,
{} as CancellationToken,
mockDbItem
);
// Should NOT have registered this database
expect(sendRequestSpy).not.to.have.been.called;
await databaseManager.removeDatabaseItem(
{} as ProgressCallback,
{} as CancellationToken,
mockDbItem
);
// Should NOT have deregistered this database
expect(sendRequestSpy).not.to.have.been.called;
expect(deregisterSpy).to.have.been.calledWith({}, {}, mockDbItem);
});
});

2
extensions/ql-vscode/src/vscode-tests/minimal-workspace/determining-selected-query-test.ts
View File

@@ -2,7 +2,7 @@ import { expect } from 'chai';
import * as path from 'path';
import * as vscode from 'vscode';
import { Uri } from 'vscode';
import { determineSelectedQuery } from '../../run-queries';
import { determineSelectedQuery } from '../../run-queries-shared';
async function showQlDocument(name: string): Promise<vscode.TextDocument> {
const folderPath = vscode.workspace.workspaceFolders![0].uri.fsPath;

2
extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/astBuilder.test.ts
View File

@@ -3,10 +3,10 @@ import { expect } from 'chai';
import * as sinon from 'sinon';
import AstBuilder from '../../../contextual/astBuilder';
import { QueryWithResults } from '../../../run-queries';
import { CodeQLCliServer } from '../../../cli';
import { DatabaseItem } from '../../../databases';
import { Uri } from 'vscode';
import { QueryWithResults } from '../../../run-queries-shared';
/**
*

14
extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts
View File

@@ -7,10 +7,8 @@ import * as sinon from 'sinon';
import { logger } from '../../logging';
import { registerQueryHistoryScubber } from '../../query-history-scrubber';
import { QueryHistoryManager, HistoryTreeDataProvider, SortOrder } from '../../query-history';
import { QueryEvaluationInfo, QueryWithResults } from '../../run-queries';
import { QueryEvaluationInfo, QueryWithResults } from '../../run-queries-shared';
import { QueryHistoryConfig, QueryHistoryConfigListener } from '../../config';
import * as messages from '../../pure/messages';
import { QueryServerClient } from '../../queryserver-client';
import { LocalQueryInfo, InitialQueryInfo } from '../../query-results';
import { DatabaseManager } from '../../databases';
import * as tmp from 'tmp-promise';
@@ -21,6 +19,7 @@ import { HistoryItemLabelProvider } from '../../history-item-label-provider';
import { RemoteQueriesManager } from '../../remote-queries/remote-queries-manager';
import { InterfaceManager } from '../../interface';
import { EvalLogViewer } from '../../eval-log-viewer';
import { QueryRunner } from '../../queryRunner';
describe('query-history', () => {
const mockExtensionLocation = path.join(tmpDir.name, 'mock-extension-location');
@@ -785,18 +784,15 @@ describe('query-history', () => {
hasInterpretedResults: () => Promise.resolve(hasInterpretedResults),
deleteQuery: sandbox.stub(),
} as unknown as QueryEvaluationInfo,
result: {
resultType: didRunSuccessfully
? messages.QueryResultType.SUCCESS
: messages.QueryResultType.OTHER_ERROR
} as messages.EvaluationResult,
sucessful: didRunSuccessfully,
message: 'foo',
dispose: sandbox.spy()
};
}
async function createMockQueryHistory(allHistory: LocalQueryInfo[]) {
const qhm = new QueryHistoryManager(
{} as QueryServerClient,
{} as QueryRunner,
{} as DatabaseManager,
localQueriesInterfaceManagerStub,
remoteQueriesManagerStub,

60
extensions/ql-vscode/src/vscode-tests/no-workspace/query-results.test.ts
View File

@@ -3,13 +3,14 @@ import * as path from 'path';
import * as fs from 'fs-extra';
import * as sinon from 'sinon';
import { LocalQueryInfo, InitialQueryInfo, interpretResultsSarif } from '../../query-results';
import { QueryEvaluationInfo, QueryWithResults } from '../../run-queries';
import { EvaluationResult, QueryResultType } from '../../pure/messages';
import { QueryWithResults } from '../../run-queries-shared';
import { DatabaseInfo, SortDirection, SortedResultSetInfo } from '../../pure/interface-types';
import { CodeQLCliServer, SourceInfo } from '../../cli';
import { CancellationTokenSource, Uri } from 'vscode';
import { tmpDir } from '../../helpers';
import { slurpQueryHistory, splatQueryHistory } from '../../query-serialization';
import { formatLegacyMessage, QueryInProgress } from '../../legacy-query-server/run-queries';
import { EvaluationResult, QueryResultType } from '../../pure/legacy-messages';
describe('query-results', () => {
let disposeSpy: sinon.SinonSpy;
@@ -87,30 +88,34 @@ describe('query-results', () => {
expect(completedQuery.getResultsPath('zxa')).to.eq('bxa');
});
it('should get the statusString', () => {
const fqi = createMockFullQueryInfo('a', createMockQueryWithResults(queryPath, false));
const completedQuery = fqi.completedQuery!;
it('should format the statusString', () => {
completedQuery.result.message = 'Tremendously';
expect(completedQuery.statusString).to.eq('failed: Tremendously');
const evalResult: EvaluationResult = {
resultType: QueryResultType.OTHER_ERROR,
evaluationTime: 12340,
queryId: 3,
runId: 1,
};
completedQuery.result.resultType = QueryResultType.OTHER_ERROR;
expect(completedQuery.statusString).to.eq('failed: Tremendously');
evalResult.message = 'Tremendously';
expect(formatLegacyMessage(evalResult)).to.eq('failed: Tremendously');
completedQuery.result.resultType = QueryResultType.CANCELLATION;
completedQuery.result.evaluationTime = 2345;
expect(completedQuery.statusString).to.eq('cancelled after 2 seconds');
evalResult.resultType = QueryResultType.OTHER_ERROR;
expect(formatLegacyMessage(evalResult)).to.eq('failed: Tremendously');
completedQuery.result.resultType = QueryResultType.OOM;
expect(completedQuery.statusString).to.eq('out of memory');
evalResult.resultType = QueryResultType.CANCELLATION;
evalResult.evaluationTime = 2345;
expect(formatLegacyMessage(evalResult)).to.eq('cancelled after 2 seconds');
completedQuery.result.resultType = QueryResultType.SUCCESS;
expect(completedQuery.statusString).to.eq('finished in 2 seconds');
evalResult.resultType = QueryResultType.OOM;
expect(formatLegacyMessage(evalResult)).to.eq('out of memory');
completedQuery.result.resultType = QueryResultType.TIMEOUT;
expect(completedQuery.statusString).to.eq('timed out after 2 seconds');
evalResult.resultType = QueryResultType.SUCCESS;
expect(formatLegacyMessage(evalResult)).to.eq('finished in 2 seconds');
evalResult.resultType = QueryResultType.TIMEOUT;
expect(formatLegacyMessage(evalResult)).to.eq('timed out after 2 seconds');
});
it('should updateSortState', async () => {
// setup
const fqi = createMockFullQueryInfo('a', createMockQueryWithResults(queryPath));
@@ -267,9 +272,6 @@ describe('query-results', () => {
if (!('quickEvalPosition' in query)) {
(query as any).quickEvalPosition = undefined;
}
if (!('templates' in query)) {
(query as any).templates = undefined;
}
}
});
expectedHistory.forEach(info => {
@@ -310,7 +312,7 @@ describe('query-results', () => {
fs.mkdirpSync(queryPath);
fs.writeFileSync(resultsPath, '', 'utf8');
const query = new QueryEvaluationInfo(
const query = new QueryInProgress(
queryPath,
Uri.file(dbPath).fsPath,
true,
@@ -321,14 +323,10 @@ describe('query-results', () => {
},
);
const result = {
query,
result: {
evaluationTime: 12340,
resultType: didRunSuccessfully
? QueryResultType.SUCCESS
: QueryResultType.OTHER_ERROR
} as EvaluationResult,
const result: QueryWithResults = {
query: query.queryEvalInfo,
sucessful: didRunSuccessfully,
message: 'foo',
dispose: disposeSpy,
};

4
extensions/ql-vscode/src/vscode-tests/no-workspace/remote-queries/remote-query-history.test.ts
View File

@@ -8,7 +8,6 @@ import { QueryHistoryConfig } from '../../../config';
import { DatabaseManager } from '../../../databases';
import { tmpDir } from '../../../helpers';
import { QueryHistoryManager } from '../../../query-history';
import { QueryServerClient } from '../../../queryserver-client';
import { Credentials } from '../../../authentication';
import { AnalysesResultsManager } from '../../../remote-queries/analyses-results-manager';
import { RemoteQueryResult } from '../../../remote-queries/shared/remote-query-result';
@@ -20,6 +19,7 @@ import { HistoryItemLabelProvider } from '../../../history-item-label-provider';
import { RemoteQueriesManager } from '../../../remote-queries/remote-queries-manager';
import { InterfaceManager } from '../../../interface';
import { EvalLogViewer } from '../../../eval-log-viewer';
import { QueryRunner } from '../../../queryRunner';
/**
* Tests for remote queries and how they interact with the query history manager.
@@ -90,7 +90,7 @@ describe('Remote queries and query history manager', function() {
remoteQueryResult1 = fs.readJSONSync(path.join(STORAGE_DIR, 'queries', rawQueryHistory[1].queryId, 'query-result.json'));
qhm = new QueryHistoryManager(
{} as QueryServerClient,
{} as QueryRunner,
{} as DatabaseManager,
localQueriesInterfaceManagerStub,
remoteQueriesManagerStub,

179
extensions/ql-vscode/src/vscode-tests/no-workspace/run-queries.test.ts
View File

@@ -4,13 +4,15 @@ import * as fs from 'fs-extra';
import * as sinon from 'sinon';
import { Uri } from 'vscode';
import { QueryEvaluationInfo } from '../../run-queries';
import { Severity, compileQuery } from '../../pure/messages';
import { Severity, compileQuery, registerDatabases, deregisterDatabases } from '../../pure/legacy-messages';
import * as config from '../../config';
import { tmpDir } from '../../helpers';
import { QueryServerClient } from '../../queryserver-client';
import { QueryServerClient } from '../../legacy-query-server/queryserver-client';
import { CodeQLCliServer } from '../../cli';
import { SELECT_QUERY_NAME } from '../../contextual/locationFinder';
import { QueryInProgress } from '../../legacy-query-server/run-queries';
import { LegacyQueryRunner } from '../../legacy-query-server/legacyRunner';
import { DatabaseItem } from '../../databases';
describe('run-queries', () => {
let sandbox: sinon.SinonSandbox;
@@ -29,40 +31,39 @@ describe('run-queries', () => {
const info = createMockQueryInfo(true, saveDir);
expect(info.compiledQueryPath).to.eq(path.join(saveDir, 'compiledQuery.qlo'));
expect(info.dilPath).to.eq(path.join(saveDir, 'results.dil'));
expect(info.resultsPaths.resultsPath).to.eq(path.join(saveDir, 'results.bqrs'));
expect(info.resultsPaths.interpretedResultsPath).to.eq(path.join(saveDir, 'interpretedResults.sarif'));
expect(info.queryEvalInfo.dilPath).to.eq(path.join(saveDir, 'results.dil'));
expect(info.queryEvalInfo.resultsPaths.resultsPath).to.eq(path.join(saveDir, 'results.bqrs'));
expect(info.queryEvalInfo.resultsPaths.interpretedResultsPath).to.eq(path.join(saveDir, 'interpretedResults.sarif'));
expect(info.dbItemPath).to.eq(Uri.file('/abc').fsPath);
});
it('should check if interpreted results can be created', async () => {
const info = createMockQueryInfo(true);
expect(info.canHaveInterpretedResults()).to.eq(true);
expect(info.queryEvalInfo.canHaveInterpretedResults(), '1').to.eq(true);
(info as any).databaseHasMetadataFile = false;
expect(info.canHaveInterpretedResults()).to.eq(false);
(info.queryEvalInfo as any).databaseHasMetadataFile = false;
expect(info.queryEvalInfo.canHaveInterpretedResults(), '2').to.eq(false);
(info as any).databaseHasMetadataFile = true;
(info.queryEvalInfo as any).databaseHasMetadataFile = true;
info.metadata!.kind = undefined;
expect(info.canHaveInterpretedResults()).to.eq(false);
expect(info.queryEvalInfo.canHaveInterpretedResults(), '3').to.eq(false);
info.metadata!.kind = 'table';
expect(info.canHaveInterpretedResults()).to.eq(false);
expect(info.queryEvalInfo.canHaveInterpretedResults(), '4').to.eq(false);
// Graphs are not interpreted unless canary is set
info.metadata!.kind = 'graph';
expect(info.canHaveInterpretedResults()).to.eq(false);
expect(info.queryEvalInfo.canHaveInterpretedResults(), '5').to.eq(false);
(config.isCanary as sinon.SinonStub).returns(true);
expect(info.canHaveInterpretedResults()).to.eq(true);
expect(info.queryEvalInfo.canHaveInterpretedResults(), '6').to.eq(true);
});
[SELECT_QUERY_NAME, 'other'].forEach(resultSetName => {
it(`should export csv results for result set ${resultSetName}`, async () => {
const csvLocation = path.join(tmpDir.name, 'test.csv');
const qs = createMockQueryServerClient(
createMockCliServer({
const cliServer = createMockCliServer({
bqrsInfo: [{ 'result-sets': [{ name: resultSetName }, { name: 'hucairz' }] }],
bqrsDecode: [{
columns: [{ kind: 'NotString' }, { kind: 'String' }],
@@ -74,10 +75,9 @@ describe('run-queries', () => {
columns: [{ kind: 'String' }, { kind: 'NotString' }, { kind: 'StillNotString' }],
tuples: [['a', 'b', 'c']]
}]
})
);
});
const info = createMockQueryInfo();
const promise = info.exportCsvResults(qs, csvLocation);
const promise = info.queryEvalInfo.exportCsvResults(cliServer, csvLocation);
const result = await promise;
expect(result).to.eq(true);
@@ -86,14 +86,14 @@ describe('run-queries', () => {
expect(csv).to.eq('a,"b"\nc,"d"\n"a",b,c\n');
// now verify that we are using the expected result set
expect((qs.cliServer.bqrsDecode as sinon.SinonStub).callCount).to.eq(2);
expect((qs.cliServer.bqrsDecode as sinon.SinonStub).getCall(0).args[1]).to.eq(resultSetName);
expect((cliServer.bqrsDecode as sinon.SinonStub).callCount).to.eq(2);
expect((cliServer.bqrsDecode as sinon.SinonStub).getCall(0).args[1]).to.eq(resultSetName);
});
});
it('should export csv results with characters that need to be escaped', async () => {
const csvLocation = path.join(tmpDir.name, 'test.csv');
const qs = createMockQueryServerClient(
const cliServer =
createMockCliServer({
bqrsInfo: [{ 'result-sets': [{ name: SELECT_QUERY_NAME }, { name: 'hucairz' }] }],
bqrsDecode: [{
@@ -109,10 +109,9 @@ describe('run-queries', () => {
[123.98, 456.99],
],
}]
})
);
});
const info = createMockQueryInfo();
const promise = info.exportCsvResults(qs, csvLocation);
const promise = info.queryEvalInfo.exportCsvResults(cliServer, csvLocation);
const result = await promise;
expect(result).to.eq(true);
@@ -121,19 +120,18 @@ describe('run-queries', () => {
expect(csv).to.eq('"a","""b"""\nc,xxx,"d,yyy"\naaa " bbb,"ccc "" ddd"\ntrue,"false"\n123,"456"\n123.98,"456.99"\n');
// now verify that we are using the expected result set
expect((qs.cliServer.bqrsDecode as sinon.SinonStub).callCount).to.eq(1);
expect((qs.cliServer.bqrsDecode as sinon.SinonStub).getCall(0).args[1]).to.eq(SELECT_QUERY_NAME);
expect((cliServer.bqrsDecode as sinon.SinonStub).callCount).to.eq(1);
expect((cliServer.bqrsDecode as sinon.SinonStub).getCall(0).args[1]).to.eq(SELECT_QUERY_NAME);
});
it('should handle csv exports for a query with no result sets', async () => {
const csvLocation = path.join(tmpDir.name, 'test.csv');
const qs = createMockQueryServerClient(
const cliServer =
createMockCliServer({
bqrsInfo: [{ 'result-sets': [] }]
})
);
});
const info = createMockQueryInfo();
const result = await info.exportCsvResults(qs, csvLocation);
const result = await info.queryEvalInfo.exportCsvResults(cliServer, csvLocation);
expect(result).to.eq(false);
});
@@ -187,9 +185,126 @@ describe('run-queries', () => {
});
});
describe('register', () => {
it('should register', async () => {
const qs = createMockQueryServerClient(
{
cliConstraints: {
supportsDatabaseRegistration: () => true
}
} as any);
const runner = new LegacyQueryRunner(qs);
const mockProgress = 'progress-monitor';
const mockCancel = 'cancel-token';
const datasetUri = Uri.file('dataset-uri');
const dbItem: DatabaseItem = {
contents: {
datasetUri
}
} as any;
await runner.registerDatabase(mockProgress as any, mockCancel as any, dbItem);
expect(qs.sendRequest).to.have.been.calledOnceWith(
registerDatabases,
{
databases: [
{
dbDir: datasetUri.fsPath,
workingSet: 'default'
}
]
},
mockCancel,
mockProgress
);
});
it('should deregister', async () => {
const qs = createMockQueryServerClient(
{
cliConstraints: {
supportsDatabaseRegistration: () => true
}
} as any);
const runner = new LegacyQueryRunner(qs);
const mockProgress = 'progress-monitor';
const mockCancel = 'cancel-token';
const datasetUri = Uri.file('dataset-uri');
const dbItem: DatabaseItem = {
contents: {
datasetUri
}
} as any;
await runner.deregisterDatabase(mockProgress as any, mockCancel as any, dbItem);
expect(qs.sendRequest).to.have.been.calledOnceWith(
deregisterDatabases,
{
databases: [
{
dbDir: datasetUri.fsPath,
workingSet: 'default'
}
]
},
mockCancel,
mockProgress
);
});
it('should not register if unsupported', async () => {
const qs = createMockQueryServerClient(
{
cliConstraints: {
supportsDatabaseRegistration: () => false
}
} as any);
const runner = new LegacyQueryRunner(qs);
const mockProgress = 'progress-monitor';
const mockCancel = 'cancel-token';
const datasetUri = Uri.file('dataset-uri');
const dbItem: DatabaseItem = {
contents: {
datasetUri
}
} as any;
await runner.registerDatabase(mockProgress as any, mockCancel as any, dbItem);
expect(qs.sendRequest).not.to.have.been.called;
});
it('should not deregister if unsupported', async () => {
const qs = createMockQueryServerClient(
{
cliConstraints: {
supportsDatabaseRegistration: () => false
}
} as any);
const runner = new LegacyQueryRunner(qs);
const mockProgress = 'progress-monitor';
const mockCancel = 'cancel-token';
const datasetUri = Uri.file('dataset-uri');
const dbItem: DatabaseItem = {
contents: {
datasetUri
}
} as any;
await runner.registerDatabase(mockProgress as any, mockCancel as any, dbItem);
expect(qs.sendRequest).not.to.have.been.called;
});
});
let queryNum = 0;
function createMockQueryInfo(databaseHasMetadataFile = true, saveDir = `save-dir${queryNum++}`) {
return new QueryEvaluationInfo(
return new QueryInProgress(
saveDir,
Uri.parse('file:///abc').fsPath,
databaseHasMetadataFile,