mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into koesie10/outcome-panel

Browse Source
This commit is contained in:
Koen Vlaswinkel
2022-09-27 14:03:13 +02:00
parent d3701944bf 0b638b6ae1
commit f408418f23
42 changed files with 2256 additions and 1703 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
extensions/ql-vscode/src/cli.ts
View File

@@ -16,7 +16,7 @@ import { DistributionProvider, FindDistributionResultKind } from './distribution
import { assertNever, getErrorMessage, getErrorStack } from './pure/helpers-pure';
import { QueryMetadata, SortDirection } from './pure/interface-types';
import { Logger, ProgressReporter } from './logging';
import { CompilationMessage } from './pure/messages';
import { CompilationMessage } from './pure/legacy-messages';
import { sarifParser } from './sarif-parser';
import { dbSchemeToLanguage, walkDirectory } from './helpers';
@@ -1248,6 +1248,9 @@ export class CliVersionConstraint {
*/
public static CLI_VERSION_WITH_LANGUAGE = new SemVer('2.4.1');
public static CLI_VERSION_WITH_NONDESTURCTIVE_UPGRADES = new SemVer('2.4.2');
/**
* CLI version where `codeql resolve upgrades` supports
* the `--allow-downgrades` flag
@@ -1264,14 +1267,6 @@ export class CliVersionConstraint {
*/
public static CLI_VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
/**
* CLI version where non destructive upgrades were introduced.
*
* This was landed in multiple parts so this is the version where all necessary feature were supported.
*/
public static CLI_VERSION_WITH_NON_DESTRUCTIVE_UPGRADES = new SemVer('2.4.2');
/**
* CLI version where the `--allow-library-packs` option to `codeql resolve queries` was
* introduced.
@@ -1351,6 +1346,10 @@ export class CliVersionConstraint {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_LANGUAGE);
}
public async supportsNonDestructiveUpgrades() {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NONDESTURCTIVE_UPGRADES);
}
public async supportsDowngrades() {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DOWNGRADES);
}
@@ -1367,10 +1366,6 @@ export class CliVersionConstraint {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DB_REGISTRATION);
}
async supportsNonDestructiveUpgrades(): Promise<boolean> {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NON_DESTRUCTIVE_UPGRADES);
}
async supportsDatabaseUnbundle() {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DATABASE_UNBUNDLE);
}

2
extensions/ql-vscode/src/contextual/astBuilder.ts
View File

@@ -1,10 +1,10 @@
import { QueryWithResults } from '../run-queries';
import { CodeQLCliServer } from '../cli';
import { DecodedBqrsChunk, BqrsId, EntityValue } from '../pure/bqrs-cli-types';
import { DatabaseItem } from '../databases';
import { ChildAstItem, AstItem } from '../astViewer';
import fileRangeFromURI from './fileRangeFromURI';
import { Uri } from 'vscode';
import { QueryWithResults } from '../run-queries-shared';
/**
* A class that wraps a tree of QL results from a query that

32
extensions/ql-vscode/src/contextual/locationFinder.ts
View File

@@ -3,13 +3,12 @@ import { ColumnKindCode, EntityValue, getResultSetSchema, ResultSetSchema } from
import { CodeQLCliServer } from '../cli';
import { DatabaseManager, DatabaseItem } from '../databases';
import fileRangeFromURI from './fileRangeFromURI';
import * as messages from '../pure/messages';
import { QueryServerClient } from '../queryserver-client';
import { QueryWithResults, compileAndRunQueryAgainstDatabase, createInitialQueryInfo } from '../run-queries';
import { ProgressCallback } from '../commandRunner';
import { KeyType } from './keyType';
import { qlpackOfDatabase, resolveQueries } from './queryResolver';
import { CancellationToken, LocationLink, Uri } from 'vscode';
import { createInitialQueryInfo, QueryWithResults } from '../run-queries-shared';
import { QueryRunner } from '../queryRunner';
export const SELECT_QUERY_NAME = '#select';
export const TEMPLATE_NAME = 'selectedSourceFile';
@@ -35,7 +34,7 @@ export interface FullLocationLink extends LocationLink {
*/
export async function getLocationsForUriString(
cli: CodeQLCliServer,
qs: QueryServerClient,
qs: QueryRunner,
dbm: DatabaseManager,
uriString: string,
keyType: KeyType,
@@ -65,19 +64,8 @@ export async function getLocationsForUriString(
},
false
);
const results = await compileAndRunQueryAgainstDatabase(
cli,
qs,
db,
initialInfo,
queryStorageDir,
progress,
token,
templates
);
if (results.result.resultType == messages.QueryResultType.SUCCESS) {
const results = await qs.compileAndRunQueryAgainstDatabase(db, initialInfo, queryStorageDir, progress, token, templates);
if (results.sucessful) {
links.push(...await getLinksFromResults(results, cli, db, filter));
}
}
@@ -114,15 +102,9 @@ async function getLinksFromResults(
return localLinks;
}
function createTemplates(path: string): messages.TemplateDefinitions {
function createTemplates(path: string): Record<string, string> {
return {
[TEMPLATE_NAME]: {
values: {
tuples: [[{
stringValue: path
}]]
}
}
[TEMPLATE_NAME]: path
};
}

44
extensions/ql-vscode/src/contextual/templateProvider.ts
View File

@@ -16,9 +16,6 @@ import { CodeQLCliServer } from '../cli';
import { DatabaseManager } from '../databases';
import { CachedOperation } from '../helpers';
import { ProgressCallback, withProgress } from '../commandRunner';
import * as messages from '../pure/messages';
import { QueryServerClient } from '../queryserver-client';
import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo, QueryWithResults } from '../run-queries';
import AstBuilder from './astBuilder';
import {
KeyType,
@@ -26,6 +23,8 @@ import {
import { FullLocationLink, getLocationsForUriString, TEMPLATE_NAME } from './locationFinder';
import { qlpackOfDatabase, resolveQueries } from './queryResolver';
import { isCanary, NO_CACHE_AST_VIEWER } from '../config';
import { createInitialQueryInfo, QueryWithResults } from '../run-queries-shared';
import { QueryRunner } from '../queryRunner';
/**
* Run templated CodeQL queries to find definitions and references in
@@ -39,7 +38,7 @@ export class TemplateQueryDefinitionProvider implements DefinitionProvider {
constructor(
private cli: CodeQLCliServer,
private qs: QueryServerClient,
private qs: QueryRunner,
private dbm: DatabaseManager,
private queryStorageDir: string,
) {
@@ -83,7 +82,7 @@ export class TemplateQueryReferenceProvider implements ReferenceProvider {
constructor(
private cli: CodeQLCliServer,
private qs: QueryServerClient,
private qs: QueryRunner,
private dbm: DatabaseManager,
private queryStorageDir: string,
) {
@@ -137,7 +136,7 @@ export class TemplatePrintAstProvider {
constructor(
private cli: CodeQLCliServer,
private qs: QueryServerClient,
private qs: QueryRunner,
private dbm: DatabaseManager,
private queryStorageDir: string,
) {
@@ -195,14 +194,9 @@ export class TemplatePrintAstProvider {
}
const query = queries[0];
const templates: messages.TemplateDefinitions = {
[TEMPLATE_NAME]: {
values: {
tuples: [[{
stringValue: zippedArchive.pathWithinSourceArchive
}]]
}
}
const templates: Record<string, string> = {
[TEMPLATE_NAME]:
zippedArchive.pathWithinSourceArchive
};
const initialInfo = await createInitialQueryInfo(
@@ -215,9 +209,7 @@ export class TemplatePrintAstProvider {
);
return {
query: await compileAndRunQueryAgainstDatabase(
this.cli,
this.qs,
query: await this.qs.compileAndRunQueryAgainstDatabase(
db,
initialInfo,
this.queryStorageDir,
@@ -231,23 +223,23 @@ export class TemplatePrintAstProvider {
}
export class TemplatePrintCfgProvider {
private cache: CachedOperation<[Uri, messages.TemplateDefinitions] | undefined>;
private cache: CachedOperation<[Uri, Record<string, string>] | undefined>;
constructor(
private cli: CodeQLCliServer,
private dbm: DatabaseManager,
) {
this.cache = new CachedOperation<[Uri, messages.TemplateDefinitions] | undefined>(this.getCfgUri.bind(this));
this.cache = new CachedOperation<[Uri, Record<string, string>] | undefined>(this.getCfgUri.bind(this));
}
async provideCfgUri(document?: TextDocument): Promise<[Uri, messages.TemplateDefinitions] | undefined> {
async provideCfgUri(document?: TextDocument): Promise<[Uri, Record<string, string>] | undefined> {
if (!document) {
return;
}
return await this.cache.get(document.uri.toString());
}
private async getCfgUri(uriString: string): Promise<[Uri, messages.TemplateDefinitions]> {
private async getCfgUri(uriString: string): Promise<[Uri, Record<string, string>]> {
const uri = Uri.parse(uriString, true);
if (uri.scheme !== zipArchiveScheme) {
throw new Error('CFG Viewing is only available for databases with zipped source archives.');
@@ -275,14 +267,8 @@ export class TemplatePrintCfgProvider {
const queryUri = Uri.file(queries[0]);
const templates: messages.TemplateDefinitions = {
[TEMPLATE_NAME]: {
values: {
tuples: [[{
stringValue: zippedArchive.pathWithinSourceArchive
}]]
}
}
const templates: Record<string, string> = {
[TEMPLATE_NAME]: zippedArchive.pathWithinSourceArchive
};
return [queryUri, templates];

22
extensions/ql-vscode/src/databases-ui.ts
View File

@@ -28,9 +28,6 @@ import {
showAndLogErrorMessage
} from './helpers';
import { logger } from './logging';
import { clearCacheInDatabase } from './run-queries';
import * as qsClient from './queryserver-client';
import { upgradeDatabaseExplicit } from './upgrades';
import {
importArchiveDatabase,
promptImportGithubDatabase,
@@ -40,6 +37,7 @@ import {
import { CancellationToken } from 'vscode';
import { asyncFilter, getErrorMessage } from './pure/helpers-pure';
import { Credentials } from './authentication';
import { QueryRunner } from './queryRunner';
import { isCanary } from './config';
type ThemableIconPath = { light: string; dark: string } | string;
@@ -220,7 +218,7 @@ export class DatabaseUI extends DisposableObject {
public constructor(
private databaseManager: DatabaseManager,
private readonly queryServer: qsClient.QueryServerClient | undefined,
private readonly queryServer: QueryRunner | undefined,
private readonly storagePath: string,
readonly extensionPath: string,
private readonly getCredentials: () => Promise<Credentials>
@@ -390,12 +388,11 @@ export class DatabaseUI extends DisposableObject {
handleChooseDatabaseFolder = async (
progress: ProgressCallback,
token: CancellationToken
): Promise<DatabaseItem | undefined> => {
): Promise<void> => {
try {
return await this.chooseAndSetDatabase(true, progress, token);
await this.chooseAndSetDatabase(true, progress, token);
} catch (e) {
void showAndLogErrorMessage(getErrorMessage(e));
return undefined;
}
};
@@ -458,12 +455,11 @@ export class DatabaseUI extends DisposableObject {
handleChooseDatabaseArchive = async (
progress: ProgressCallback,
token: CancellationToken
): Promise<DatabaseItem | undefined> => {
): Promise<void> => {
try {
return await this.chooseAndSetDatabase(false, progress, token);
await this.chooseAndSetDatabase(false, progress, token);
} catch (e) {
void showAndLogErrorMessage(getErrorMessage(e));
return undefined;
}
};
@@ -576,8 +572,7 @@ export class DatabaseUI extends DisposableObject {
// Search for upgrade scripts in any workspace folders available
await upgradeDatabaseExplicit(
this.queryServer,
await this.queryServer.upgradeDatabaseExplicit(
databaseItem,
progress,
token
@@ -592,8 +587,7 @@ export class DatabaseUI extends DisposableObject {
this.queryServer !== undefined &&
this.databaseManager.currentDatabaseItem !== undefined
) {
await clearCacheInDatabase(
this.queryServer,
await this.queryServer.clearCacheInDatabase(
this.databaseManager.currentDatabaseItem,
progress,
token

24
extensions/ql-vscode/src/databases.ts
View File

@@ -17,9 +17,8 @@ import {
import { zipArchiveScheme, encodeArchiveBasePath, decodeSourceArchiveUri, encodeSourceArchiveUri } from './archive-filesystem-provider';
import { DisposableObject } from './pure/disposable-object';
import { Logger, logger } from './logging';
import { registerDatabases, Dataset, deregisterDatabases } from './pure/messages';
import { QueryServerClient } from './queryserver-client';
import { getErrorMessage } from './pure/helpers-pure';
import { QueryRunner } from './queryRunner';
/**
* databases.ts
@@ -555,13 +554,13 @@ export class DatabaseManager extends DisposableObject {
constructor(
private readonly ctx: ExtensionContext,
private readonly qs: QueryServerClient,
private readonly qs: QueryRunner,
private readonly cli: cli.CodeQLCliServer,
public logger: Logger
) {
super();
qs.onDidStartQueryServer(this.reregisterDatabases.bind(this));
qs.onStart(this.reregisterDatabases.bind(this));
// Let this run async.
void this.loadPersistedState();
@@ -860,27 +859,14 @@ export class DatabaseManager extends DisposableObject {
token: vscode.CancellationToken,
dbItem: DatabaseItem,
) {
if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
const databases: Dataset[] = [{
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default'
}];
await this.qs.sendRequest(deregisterDatabases, { databases }, token, progress);
}
await this.qs.deregisterDatabase(progress, token, dbItem);
}
private async registerDatabase(
progress: ProgressCallback,
token: vscode.CancellationToken,
dbItem: DatabaseItem,
) {
if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
const databases: Dataset[] = [{
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default'
}];
await this.qs.sendRequest(registerDatabases, { databases }, token, progress);
}
await this.qs.registerDatabase(progress, token, dbItem);
}
private updatePersistedCurrentDatabaseItem(): void {

51
extensions/ql-vscode/src/extension.ts
View File

@@ -70,12 +70,11 @@ import { asError, assertNever, getErrorMessage } from './pure/helpers-pure';
import { spawnIdeServer } from './ide-server';
import { ResultsView } from './interface';
import { WebviewReveal } from './interface-utils';
import { ideServerLogger, logger, queryServerLogger } from './logging';
import { ideServerLogger, logger, ProgressReporter, queryServerLogger } from './logging';
import { QueryHistoryManager } from './query-history';
import { CompletedLocalQueryInfo, LocalQueryInfo } from './query-results';
import * as qsClient from './queryserver-client';
import * as qsClient from './legacy-query-server/queryserver-client';
import { displayQuickQuery } from './quick-query';
import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo } from './run-queries';
import { QLTestAdapterFactory } from './test-adapter';
import { TestUIService } from './test-ui';
import { CompareView } from './compare/compare-view';
@@ -102,6 +101,9 @@ import { EvalLogViewer } from './eval-log-viewer';
import { SummaryLanguageSupport } from './log-insights/summary-language-support';
import { JoinOrderScannerProvider } from './log-insights/join-order';
import { LogScannerService } from './log-insights/log-scanner-service';
import { createInitialQueryInfo } from './run-queries-shared';
import { LegacyQueryRunner } from './legacy-query-server/legacyRunner';
import { QueryRunner } from './queryRunner';
import { VariantAnalysisView } from './remote-queries/variant-analysis-view';
/**
@@ -165,7 +167,7 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
export interface CodeQLExtensionInterface {
readonly ctx: ExtensionContext;
readonly cliServer: CodeQLCliServer;
readonly qs: qsClient.QueryServerClient;
readonly qs: QueryRunner;
readonly distributionManager: DistributionManager;
readonly databaseManager: DatabaseManager;
readonly databaseUI: DatabaseUI;
@@ -417,21 +419,7 @@ async function activateWithInstalledDistribution(
ctx.subscriptions.push(statusBar);
void logger.log('Initializing query server client.');
const qs = new qsClient.QueryServerClient(
qlConfigurationListener,
cliServer,
{
logger: queryServerLogger,
contextStoragePath: getContextStoragePath(ctx),
},
(task) =>
Window.withProgress(
{ title: 'CodeQL query server', location: ProgressLocation.Window },
task
)
);
ctx.subscriptions.push(qs);
await qs.startQueryServer();
const qs = await createQueryServer(qlConfigurationListener, cliServer, ctx);
void logger.log('Initializing database manager.');
const dbm = new DatabaseManager(ctx, qs, cliServer, logger);
@@ -553,9 +541,7 @@ async function activateWithInstalledDistribution(
const item = new LocalQueryInfo(initialInfo, source);
qhm.addQuery(item);
try {
const completedQueryInfo = await compileAndRunQueryAgainstDatabase(
cliServer,
qs,
const completedQueryInfo = await qs.compileAndRunQueryAgainstDatabase(
databaseItem,
initialInfo,
queryStorageDir,
@@ -789,6 +775,7 @@ async function activateWithInstalledDistribution(
// requests for each query to run.
// Only do it if running multiple queries since this check is
// performed on each query run anyway.
// Don't do this with non destructive upgrades as the user won't see anything anyway.
await databaseUI.tryUpgradeCurrentDatabase(progress, token);
}
@@ -1147,6 +1134,26 @@ async function activateWithInstalledDistribution(
};
}
async function createQueryServer(qlConfigurationListener: QueryServerConfigListener, cliServer: CodeQLCliServer, ctx: ExtensionContext): Promise<QueryRunner> {
const qsOpts = {
logger: queryServerLogger,
contextStoragePath: getContextStoragePath(ctx),
};
const progressCallback = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable<void>) => Window.withProgress(
{ title: 'CodeQL query server', location: ProgressLocation.Window },
task
);
const qs = new qsClient.QueryServerClient(
qlConfigurationListener,
cliServer,
qsOpts,
progressCallback
);
ctx.subscriptions.push(qs);
await qs.startQueryServer();
return new LegacyQueryRunner(qs);
}
function getContextStoragePath(ctx: ExtensionContext) {
return ctx.storageUri?.fsPath || ctx.globalStorageUri.fsPath;
}

5
extensions/ql-vscode/src/interface.ts
View File

@@ -29,10 +29,9 @@ import {
RawResultsSortState,
} from './pure/interface-types';
import { Logger } from './logging';
import * as messages from './pure/messages';
import { commandRunner } from './commandRunner';
import { CompletedQueryInfo, interpretResultsSarif, interpretGraphResults } from './query-results';
import { QueryEvaluationInfo } from './run-queries';
import { QueryEvaluationInfo } from './run-queries-shared';
import { parseSarifLocation, parseSarifPlainTextMessage } from './pure/sarif-utils';
import {
WebviewReveal,
@@ -327,7 +326,7 @@ export class ResultsView extends AbstractWebview<IntoResultsViewMsg, FromResults
forceReveal: WebviewReveal,
shouldKeepOldResultsWhileRendering = false
): Promise<void> {
if (fullQuery.completedQuery.result.resultType !== messages.QueryResultType.SUCCESS) {
if (!fullQuery.completedQuery.sucessful) {
return;
}

30
extensions/ql-vscode/src/json-rpc-server.ts Normal file
View File

@@ -0,0 +1,30 @@
import { Logger } from './logging';
import * as cp from 'child_process';
import { Disposable } from 'vscode';
import { MessageConnection } from 'vscode-jsonrpc';
/** A running query server process and its associated message connection. */
export class ServerProcess implements Disposable {
child: cp.ChildProcess;
connection: MessageConnection;
logger: Logger;
constructor(child: cp.ChildProcess, connection: MessageConnection, private name: string, logger: Logger) {
this.child = child;
this.connection = connection;
this.logger = logger;
}
dispose(): void {
void this.logger.log(`Stopping ${this.name}...`);
this.connection.dispose();
this.child.stdin!.end();
this.child.stderr!.destroy();
// TODO kill the process if it doesn't terminate after a certain time limit.
// On Windows, we usually have to terminate the process before closing its stdout.
this.child.stdout!.destroy();
void this.logger.log(`Stopped ${this.name}.`);
}
}

59
extensions/ql-vscode/src/legacy-query-server/legacyRunner.ts Normal file
View File

@@ -0,0 +1,59 @@
import { CancellationToken } from 'vscode';
import { ProgressCallback } from '../commandRunner';
import { DatabaseItem } from '../databases';
import { Dataset, deregisterDatabases, registerDatabases } from '../pure/legacy-messages';
import { InitialQueryInfo, LocalQueryInfo } from '../query-results';
import { QueryRunner } from '../queryRunner';
import { QueryWithResults } from '../run-queries-shared';
import { QueryServerClient } from './queryserver-client';
import { clearCacheInDatabase, compileAndRunQueryAgainstDatabase } from './run-queries';
import { upgradeDatabaseExplicit } from './upgrades';
export class LegacyQueryRunner extends QueryRunner {
constructor(public readonly qs: QueryServerClient) {
super();
}
get cliServer() {
return this.qs.cliServer;
}
async restartQueryServer(progress: ProgressCallback, token: CancellationToken): Promise<void> {
await this.qs.restartQueryServer(progress, token);
}
onStart(callBack: (progress: ProgressCallback, token: CancellationToken) => Promise<void>) {
this.qs.onDidStartQueryServer(callBack);
}
async clearCacheInDatabase(dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken): Promise<void> {
await clearCacheInDatabase(this.qs, dbItem, progress, token);
}
async compileAndRunQueryAgainstDatabase(dbItem: DatabaseItem, initialInfo: InitialQueryInfo, queryStorageDir: string, progress: ProgressCallback, token: CancellationToken, templates?: Record<string, string>, queryInfo?: LocalQueryInfo): Promise<QueryWithResults> {
return await compileAndRunQueryAgainstDatabase(this.qs.cliServer, this.qs, dbItem, initialInfo, queryStorageDir, progress, token, templates, queryInfo);
}
async deregisterDatabase(progress: ProgressCallback, token: CancellationToken, dbItem: DatabaseItem): Promise<void> {
if (dbItem.contents && (await this.qs.cliServer.cliConstraints.supportsDatabaseRegistration())) {
const databases: Dataset[] = [{
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default'
}];
await this.qs.sendRequest(deregisterDatabases, { databases }, token, progress);
}
}
async registerDatabase(progress: ProgressCallback, token: CancellationToken, dbItem: DatabaseItem): Promise<void> {
if (dbItem.contents && (await this.qs.cliServer.cliConstraints.supportsDatabaseRegistration())) {
const databases: Dataset[] = [{
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default'
}];
await this.qs.sendRequest(registerDatabases, { databases }, token, progress);
}
}
async upgradeDatabaseExplicit(dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken): Promise<void> {
await upgradeDatabaseExplicit(this.qs, dbItem, progress, token);
}
}

76
extensions/ql-vscode/src/queryserver-client.ts → extensions/ql-vscode/src/legacy-query-server/queryserver-client.ts
View File

@@ -1,49 +1,25 @@
import * as cp from 'child_process';
import * as path from 'path';
import * as fs from 'fs-extra';
import { DisposableObject } from './pure/disposable-object';
import { Disposable, CancellationToken, commands } from 'vscode';
import { createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc';
import * as cli from './cli';
import { QueryServerConfig } from './config';
import { Logger, ProgressReporter } from './logging';
import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from './pure/messages';
import * as messages from './pure/messages';
import { ProgressCallback, ProgressTask } from './commandRunner';
import { DisposableObject } from '../pure/disposable-object';
import { CancellationToken, commands } from 'vscode';
import { createMessageConnection, RequestType } from 'vscode-jsonrpc';
import * as cli from '../cli';
import { QueryServerConfig } from '../config';
import { Logger, ProgressReporter } from '../logging';
import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from '../pure/legacy-messages';
import * as messages from '../pure/legacy-messages';
import { ProgressCallback, ProgressTask } from '../commandRunner';
import { findQueryLogFile } from '../run-queries-shared';
import { ServerProcess } from '../json-rpc-server';
type WithProgressReporting = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable<void>) => Thenable<void>;
type ServerOpts = {
logger: Logger;
contextStoragePath: string;
}
/** A running query server process and its associated message connection. */
class ServerProcess implements Disposable {
child: cp.ChildProcess;
connection: MessageConnection;
logger: Logger;
constructor(child: cp.ChildProcess, connection: MessageConnection, logger: Logger) {
this.child = child;
this.connection = connection;
this.logger = logger;
}
dispose(): void {
void this.logger.log('Stopping query server...');
this.connection.dispose();
this.child.stdin!.end();
this.child.stderr!.destroy();
// TODO kill the process if it doesn't terminate after a certain time limit.
// On Windows, we usually have to terminate the process before closing its stdout.
this.child.stdout!.destroy();
void this.logger.log('Stopped query server.');
}
}
type WithProgressReporting = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable<void>) => Thenable<void>;
/**
* Client that manages a query server process.
* The server process is started upon initialization and tracked during its lifetime.
@@ -200,7 +176,7 @@ export class QueryServerClient extends DisposableObject {
callback(res);
}
});
this.serverProcess = new ServerProcess(child, connection, this.logger);
this.serverProcess = new ServerProcess(child, connection, 'Query server', this.logger);
// Ensure the server process is disposed together with this client.
this.track(this.serverProcess);
connection.listen();
@@ -254,27 +230,3 @@ export class QueryServerClient extends DisposableObject {
}
}
}
export function findQueryLogFile(resultPath: string): string {
return path.join(resultPath, 'query.log');
}
export function findQueryEvalLogFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.jsonl');
}
export function findQueryEvalLogSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary');
}
export function findJsonQueryEvalLogSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary.jsonl');
}
export function findQueryEvalLogSummarySymbolsFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary.symbols.json');
}
export function findQueryEvalLogEndSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log-end.summary');
}

518
extensions/ql-vscode/src/legacy-query-server/run-queries.ts Normal file
View File

@@ -0,0 +1,518 @@
import * as crypto from 'crypto';
import * as fs from 'fs-extra';
import * as tmp from 'tmp-promise';
import * as path from 'path';
import {
CancellationToken,
Uri,
} from 'vscode';
import { ErrorCodes, ResponseError } from 'vscode-languageclient';
import * as cli from '../cli';
import { DatabaseItem, } from '../databases';
import {
getOnDiskWorkspaceFolders,
showAndLogErrorMessage,
showAndLogWarningMessage,
tryGetQueryMetadata,
upgradesTmpDir
} from '../helpers';
import { ProgressCallback } from '../commandRunner';
import { QueryMetadata } from '../pure/interface-types';
import { logger } from '../logging';
import * as messages from '../pure/legacy-messages';
import { InitialQueryInfo, LocalQueryInfo } from '../query-results';
import * as qsClient from './queryserver-client';
import { getErrorMessage } from '../pure/helpers-pure';
import { compileDatabaseUpgradeSequence, upgradeDatabaseExplicit } from './upgrades';
import { QueryEvaluationInfo, QueryWithResults } from '../run-queries-shared';
/**
* A collection of evaluation-time information about a query,
* including the query itself, and where we have decided to put
* temporary files associated with it, such as the compiled query
* output and results.
*/
export class QueryInProgress {
public queryEvalInfo: QueryEvaluationInfo;
/**
* Note that in the {@link slurpQueryHistory} method, we create a QueryEvaluationInfo instance
* by explicitly setting the prototype in order to avoid calling this constructor.
*/
constructor(
readonly querySaveDir: string,
readonly dbItemPath: string,
databaseHasMetadataFile: boolean,
readonly queryDbscheme: string, // the dbscheme file the query expects, based on library path resolution
readonly quickEvalPosition?: messages.Position,
readonly metadata?: QueryMetadata,
readonly templates?: Record<string, string>,
) {
this.queryEvalInfo = new QueryEvaluationInfo(querySaveDir, dbItemPath, databaseHasMetadataFile, quickEvalPosition, metadata);
/**/
}
get compiledQueryPath() {
return path.join(this.querySaveDir, 'compiledQuery.qlo');
}
async run(
qs: qsClient.QueryServerClient,
upgradeQlo: string | undefined,
availableMlModels: cli.MlModelInfo[],
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
queryInfo?: LocalQueryInfo,
): Promise<messages.EvaluationResult> {
if (!dbItem.contents || dbItem.error) {
throw new Error('Can\'t run query on invalid database.');
}
let result: messages.EvaluationResult | null = null;
const callbackId = qs.registerCallback(res => {
result = {
...res,
logFileLocation: this.queryEvalInfo.logPath
};
});
const availableMlModelUris: messages.MlModel[] = availableMlModels.map(model => ({ uri: Uri.file(model.path).toString(true) }));
const queryToRun: messages.QueryToRun = {
resultsPath: this.queryEvalInfo.resultsPaths.resultsPath,
qlo: Uri.file(this.compiledQueryPath).toString(),
compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(),
allowUnknownTemplates: true,
templateValues: createSimpleTemplates(this.templates),
availableMlModels: availableMlModelUris,
id: callbackId,
timeoutSecs: qs.config.timeoutSecs,
};
const dataset: messages.Dataset = {
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default'
};
if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
await qs.sendRequest(messages.startLog, {
db: dataset,
logPath: this.queryEvalInfo.evalLogPath,
});
}
const params: messages.EvaluateQueriesParams = {
db: dataset,
evaluateId: callbackId,
queries: [queryToRun],
stopOnError: false,
useSequenceHint: false
};
try {
await qs.sendRequest(messages.runQueries, params, token, progress);
if (qs.config.customLogDirectory) {
void showAndLogWarningMessage(
`Custom log directories are no longer supported. The "codeQL.runningQueries.customLogDirectory" setting is deprecated. Unset the setting to stop seeing this message. Query logs saved to ${this.queryEvalInfo.logPath}.`
);
}
} finally {
qs.unRegisterCallback(callbackId);
if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
await qs.sendRequest(messages.endLog, {
db: dataset,
logPath: this.queryEvalInfo.evalLogPath,
});
if (await this.queryEvalInfo.hasEvalLog()) {
await this.queryEvalInfo.addQueryLogs(queryInfo, qs.cliServer, qs.logger);
} else {
void showAndLogWarningMessage(`Failed to write structured evaluator log to ${this.queryEvalInfo.evalLogPath}.`);
}
}
}
return result || {
evaluationTime: 0,
message: 'No result from server',
queryId: -1,
runId: callbackId,
resultType: messages.QueryResultType.OTHER_ERROR
};
}
async compile(
qs: qsClient.QueryServerClient,
program: messages.QlProgram,
progress: ProgressCallback,
token: CancellationToken,
): Promise<messages.CompilationMessage[]> {
let compiled: messages.CheckQueryResult | undefined;
try {
const target = this.quickEvalPosition ? {
quickEval: { quickEvalPos: this.quickEvalPosition }
} : { query: {} };
const params: messages.CompileQueryParams = {
compilationOptions: {
computeNoLocationUrls: true,
failOnWarnings: false,
fastCompilation: false,
includeDilInQlo: true,
localChecking: false,
noComputeGetUrl: false,
noComputeToString: false,
computeDefaultStrings: true,
emitDebugInfo: true
},
extraOptions: {
timeoutSecs: qs.config.timeoutSecs
},
queryToCheck: program,
resultPath: this.compiledQueryPath,
target,
};
compiled = await qs.sendRequest(messages.compileQuery, params, token, progress);
} finally {
void qs.logger.log(' - - - COMPILATION DONE - - - ', { additionalLogLocation: this.queryEvalInfo.logPath });
}
return (compiled?.messages || []).filter(msg => msg.severity === messages.Severity.ERROR);
}
}
export async function clearCacheInDatabase(
qs: qsClient.QueryServerClient,
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<messages.ClearCacheResult> {
if (dbItem.contents === undefined) {
throw new Error('Can\'t clear the cache in an invalid database.');
}
const db: messages.Dataset = {
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default',
};
const params: messages.ClearCacheParams = {
dryRun: false,
db,
};
return qs.sendRequest(messages.clearCache, params, token, progress);
}
/**
* Compare the dbscheme implied by the query `query` and that of the current database.
* - If they are compatible, do nothing.
* - If they are incompatible but the database can be upgraded, suggest that upgrade.
* - If they are incompatible and the database cannot be upgraded, throw an error.
*/
async function checkDbschemeCompatibility(
cliServer: cli.CodeQLCliServer,
qs: qsClient.QueryServerClient,
query: QueryInProgress,
qlProgram: messages.QlProgram,
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<void> {
const searchPath = getOnDiskWorkspaceFolders();
if (dbItem.contents?.dbSchemeUri !== undefined) {
const { finalDbscheme } = await cliServer.resolveUpgrades(dbItem.contents.dbSchemeUri.fsPath, searchPath, false);
const hash = async function(filename: string): Promise<string> {
return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex');
};
// At this point, we have learned about three dbschemes:
// the dbscheme of the actual database we're querying.
const dbschemeOfDb = await hash(dbItem.contents.dbSchemeUri.fsPath);
// the dbscheme of the query we're running, including the library we've resolved it to use.
const dbschemeOfLib = await hash(query.queryDbscheme);
// the database we're able to upgrade to
const upgradableTo = await hash(finalDbscheme);
if (upgradableTo != dbschemeOfLib) {
reportNoUpgradePath(qlProgram, query);
}
if (upgradableTo == dbschemeOfLib &&
dbschemeOfDb != dbschemeOfLib) {
// Try to upgrade the database
await upgradeDatabaseExplicit(
qs,
dbItem,
progress,
token
);
}
}
}
function reportNoUpgradePath(qlProgram: messages.QlProgram, query: QueryInProgress): void {
throw new Error(
`Query ${qlProgram.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.`
);
}
/**
* Compile a non-destructive upgrade.
*/
async function compileNonDestructiveUpgrade(
qs: qsClient.QueryServerClient,
upgradeTemp: tmp.DirectoryResult,
query: QueryInProgress,
qlProgram: messages.QlProgram,
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<string> {
if (!dbItem?.contents?.dbSchemeUri) {
throw new Error('Database is invalid, and cannot be upgraded.');
}
// When packaging is used, dependencies may exist outside of the workspace and they are always on the resolved search path.
// When packaging is not used, all dependencies are in the workspace.
const upgradesPath = (await qs.cliServer.cliConstraints.supportsPackaging())
? qlProgram.libraryPath
: getOnDiskWorkspaceFolders();
const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(
dbItem.contents.dbSchemeUri.fsPath,
upgradesPath,
true,
query.queryDbscheme
);
if (!matchesTarget) {
reportNoUpgradePath(qlProgram, query);
}
const result = await compileDatabaseUpgradeSequence(qs, dbItem, scripts, upgradeTemp, progress, token);
if (result.compiledUpgrade === undefined) {
const error = result.error || '[no error message available]';
throw new Error(error);
}
// We can upgrade to the actual target
qlProgram.dbschemePath = query.queryDbscheme;
// We are new enough that we will always support single file upgrades.
return result.compiledUpgrade;
}
export async function compileAndRunQueryAgainstDatabase(
cliServer: cli.CodeQLCliServer,
qs: qsClient.QueryServerClient,
dbItem: DatabaseItem,
initialInfo: InitialQueryInfo,
queryStorageDir: string,
progress: ProgressCallback,
token: CancellationToken,
templates?: Record<string, string>,
queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
): Promise<QueryWithResults> {
if (!dbItem.contents || !dbItem.contents.dbSchemeUri) {
throw new Error(`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`);
}
// Get the workspace folder paths.
const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
// Figure out the library path for the query.
const packConfig = await cliServer.resolveLibraryPath(diskWorkspaceFolders, initialInfo.queryPath);
if (!packConfig.dbscheme) {
throw new Error('Could not find a database scheme for this query. Please check that you have a valid qlpack.yml file for this query, which refers to a database scheme either in the `dbscheme` field or through one of its dependencies.');
}
// Check whether the query has an entirely different schema from the
// database. (Queries that merely need the database to be upgraded
// won't trigger this check)
// This test will produce confusing results if we ever change the name of the database schema files.
const querySchemaName = path.basename(packConfig.dbscheme);
const dbSchemaName = path.basename(dbItem.contents.dbSchemeUri.fsPath);
if (querySchemaName != dbSchemaName) {
void logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`);
throw new Error(`The query ${path.basename(initialInfo.queryPath)} cannot be run against the selected database (${dbItem.name}): their target languages are different. Please select a different database and try again.`);
}
const qlProgram: messages.QlProgram = {
// The project of the current document determines which library path
// we use. The `libraryPath` field in this server message is relative
// to the workspace root, not to the project root.
libraryPath: packConfig.libraryPath,
// Since we are compiling and running a query against a database,
// we use the database's DB scheme here instead of the DB scheme
// from the current document's project.
dbschemePath: dbItem.contents.dbSchemeUri.fsPath,
queryPath: initialInfo.queryPath
};
// Read the query metadata if possible, to use in the UI.
const metadata = await tryGetQueryMetadata(cliServer, qlProgram.queryPath);
let availableMlModels: cli.MlModelInfo[] = [];
if (!await cliServer.cliConstraints.supportsResolveMlModels()) {
void logger.log('Resolving ML models is unsupported by this version of the CLI. Running the query without any ML models.');
} else {
try {
availableMlModels = (await cliServer.resolveMlModels(diskWorkspaceFolders, initialInfo.queryPath)).models;
if (availableMlModels.length) {
void logger.log(`Found available ML models at the following paths: ${availableMlModels.map(x => `'${x.path}'`).join(', ')}.`);
} else {
void logger.log('Did not find any available ML models.');
}
} catch (e) {
const message = `Couldn't resolve available ML models for ${qlProgram.queryPath}. Running the ` +
`query without any ML models: ${e}.`;
void showAndLogErrorMessage(message);
}
}
const hasMetadataFile = (await dbItem.hasMetadataFile());
const query = new QueryInProgress(
path.join(queryStorageDir, initialInfo.id),
dbItem.databaseUri.fsPath,
hasMetadataFile,
packConfig.dbscheme,
initialInfo.quickEvalPosition,
metadata,
templates
);
await query.queryEvalInfo.createTimestampFile();
let upgradeDir: tmp.DirectoryResult | undefined;
try {
let upgradeQlo;
if (await cliServer.cliConstraints.supportsNonDestructiveUpgrades()) {
upgradeDir = await tmp.dir({ dir: upgradesTmpDir, unsafeCleanup: true });
upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, qlProgram, dbItem, progress, token);
} else {
await checkDbschemeCompatibility(cliServer, qs, query, qlProgram, dbItem, progress, token);
}
let errors;
try {
errors = await query.compile(qs, qlProgram, progress, token);
} catch (e) {
if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
return createSyntheticResult(query, 'Query cancelled');
} else {
throw e;
}
}
if (errors.length === 0) {
const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token, queryInfo);
if (result.resultType !== messages.QueryResultType.SUCCESS) {
const message = result.message || 'Failed to run query';
void logger.log(message);
void showAndLogErrorMessage(message);
}
const message = formatLegacyMessage(result);
return {
query: query.queryEvalInfo,
message,
result,
sucessful: result.resultType == messages.QueryResultType.SUCCESS,
logFileLocation: result.logFileLocation,
dispose: () => {
qs.logger.removeAdditionalLogLocation(result.logFileLocation);
}
};
} else {
// Error dialogs are limited in size and scrollability,
// so we include a general description of the problem,
// and direct the user to the output window for the detailed compilation messages.
// However we don't show quick eval errors there so we need to display them anyway.
void qs.logger.log(
`Failed to compile query ${initialInfo.queryPath} against database scheme ${qlProgram.dbschemePath}:`,
{ additionalLogLocation: query.queryEvalInfo.logPath }
);
const formattedMessages: string[] = [];
for (const error of errors) {
const message = error.message || '[no error message available]';
const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
formattedMessages.push(formatted);
void qs.logger.log(formatted, { additionalLogLocation: query.queryEvalInfo.logPath });
}
if (initialInfo.isQuickEval && formattedMessages.length <= 2) {
// If there are more than 2 error messages, they will not be displayed well in a popup
// and will be trimmed by the function displaying the error popup. Accordingly, we only
// try to show the errors if there are 2 or less, otherwise we direct the user to the log.
void showAndLogErrorMessage('Quick evaluation compilation failed: ' + formattedMessages.join('\n'));
} else {
void showAndLogErrorMessage((initialInfo.isQuickEval ? 'Quick evaluation' : 'Query') + compilationFailedErrorTail);
}
return createSyntheticResult(query, 'Query had compilation errors');
}
} finally {
try {
await upgradeDir?.cleanup();
} catch (e) {
void qs.logger.log(
`Could not clean up the upgrades dir. Reason: ${getErrorMessage(e)}`,
{ additionalLogLocation: query.queryEvalInfo.logPath }
);
}
}
}
const compilationFailedErrorTail = ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
' and the query and database use the same target language. For more details on the error, go to View > Output,' +
' and choose CodeQL Query Server from the dropdown.';
export function formatLegacyMessage(result: messages.EvaluationResult) {
switch (result.resultType) {
case messages.QueryResultType.CANCELLATION:
return `cancelled after ${Math.round(result.evaluationTime / 1000)} seconds`;
case messages.QueryResultType.OOM:
return 'out of memory';
case messages.QueryResultType.SUCCESS:
return `finished in ${Math.round(result.evaluationTime / 1000)} seconds`;
case messages.QueryResultType.TIMEOUT:
return `timed out after ${Math.round(result.evaluationTime / 1000)} seconds`;
case messages.QueryResultType.OTHER_ERROR:
default:
return result.message ? `failed: ${result.message}` : 'failed';
}
}
/**
* Create a synthetic result for a query that failed to compile.
*/
function createSyntheticResult(
query: QueryInProgress,
message: string,
): QueryWithResults {
return {
query: query.queryEvalInfo,
message,
sucessful: false,
dispose: () => { /**/ },
};
}
function createSimpleTemplates(templates: Record<string, string> | undefined): messages.TemplateDefinitions | undefined {
if (!templates) {
return undefined;
}
const result: messages.TemplateDefinitions = {};
for (const key of Object.keys(templates)) {
result[key] = {
values: {
tuples: [[{ stringValue: templates[key] }]]
}
};
}
return result;
}

13
extensions/ql-vscode/src/upgrades.ts → extensions/ql-vscode/src/legacy-query-server/upgrades.ts
View File

@@ -1,12 +1,12 @@
import * as vscode from 'vscode';
import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, tmpDir } from './helpers';
import { ProgressCallback, UserCancellationException } from './commandRunner';
import { logger } from './logging';
import * as messages from './pure/messages';
import * as qsClient from './queryserver-client';
import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, tmpDir } from '../helpers';
import { ProgressCallback, UserCancellationException } from '../commandRunner';
import { logger } from '../logging';
import * as messages from '../pure/legacy-messages';
import * as qsClient from '../legacy-query-server/queryserver-client';
import * as tmp from 'tmp-promise';
import * as path from 'path';
import { DatabaseItem } from './databases';
import { DatabaseItem } from '../databases';
/**
* Maximum number of lines to include from database upgrade message,
@@ -16,7 +16,6 @@ import { DatabaseItem } from './databases';
const MAX_UPGRADE_MESSAGE_LINES = 10;
/**
* Compile a database upgrade sequence.
* Callers must check that this is valid with the current queryserver first.

4
extensions/ql-vscode/src/log-insights/join-order.ts
View File

@@ -7,8 +7,8 @@ const DEFAULT_WARNING_THRESHOLD = 50;
/**
* Like `max`, but returns 0 if no meaningful maximum can be computed.
*/
function safeMax(it: Iterable<number>) {
const m = Math.max(...it);
function safeMax(it?: Iterable<number>) {
const m = Math.max(...(it || []));
return Number.isFinite(m) ? m : 0;
}

101
extensions/ql-vscode/src/pure/messages.ts → extensions/ql-vscode/src/pure/legacy-messages.ts
View File

@@ -15,38 +15,7 @@
*/
import * as rpc from 'vscode-jsonrpc';
/**
* A position within a QL file.
*/
export interface Position {
/**
* The one-based index of the start line
*/
line: number;
/**
* The one-based offset of the start column within
* the start line in UTF-16 code-units
*/
column: number;
/**
* The one-based index of the end line line
*/
endLine: number;
/**
* The one-based offset of the end column within
* the end line in UTF-16 code-units
*/
endColumn: number;
/**
* The path of the file.
* If the file name is "Compiler Generated" the
* the position is not a real position but
* arises from compiler generated code.
*/
fileName: string;
}
import * as shared from './messages-shared';
/**
* A query that should be checked for any errors or warnings
@@ -258,28 +227,6 @@ export interface DILQuery {
dilSource: string;
}
/**
* The way of compiling the query, as a normal query
* or a subset of it. Note that precisely one of the two options should be set.
*/
export interface CompilationTarget {
/**
* Compile as a normal query
*/
query?: Record<string, never>;
/**
* Compile as a quick evaluation
*/
quickEval?: QuickEvalOptions;
}
/**
* Options for quick evaluation
*/
export interface QuickEvalOptions {
quickEvalPos?: Position;
}
/**
* The result of checking a query.
*/
@@ -1012,37 +959,20 @@ export type DeregisterDatabasesResult = {
};
/**
* Type for any action that could have progress messages.
* A position within a QL file.
*/
export interface WithProgressId<T> {
/**
* The main body
*/
body: T;
/**
* The id used to report progress updates
*/
progressId: number;
}
export type Position = shared.Position;
export interface ProgressMessage {
/**
* The id of the operation that is running
*/
id: number;
/**
* The current step
*/
step: number;
/**
* The maximum step. This *should* be constant for a single job.
*/
maxStep: number;
/**
* The current progress message
*/
message: string;
}
/**
* The way of compiling the query, as a normal query
* or a subset of it. Note that precisely one of the two options should be set.
*/
export type CompilationTarget = shared.CompilationTarget;
export type QuickEvalOptions = shared.QuickEvalOptions;
export type WithProgressId<T> = shared.WithProgressId<T>;
export type ProgressMessage = shared.ProgressMessage;
/**
* Check a Ql query for errors without compiling it
@@ -1120,7 +1050,4 @@ export const deregisterDatabases = new rpc.RequestType<
*/
export const completeQuery = new rpc.RequestType<EvaluationResult, Record<string, any>, void, void>('evaluation/queryCompleted');
/**
* A notification that the progress has been changed.
*/
export const progress = new rpc.NotificationType<ProgressMessage, void>('ql/progressUpdated');
export const progress = shared.progress;

110
extensions/ql-vscode/src/pure/messages-shared.ts Normal file
View File

@@ -0,0 +1,110 @@
/**
* Types for messages exchanged during jsonrpc communication with the
* the CodeQL query server.
*
* This file exists in the queryserver and in the vscode extension, and
* should be kept in sync between them.
*
* A note about the namespaces below, which look like they are
* essentially enums, namely Severity, ResultColumnKind, and
* QueryResultType. By design, for the sake of extensibility, clients
* receiving messages of this protocol are supposed to accept any
* number for any of these types. We commit to the given meaning of
* the numbers listed in constants in the namespaces, and we commit to
* the fact that any unknown QueryResultType value counts as an error.
*/
import * as rpc from 'vscode-jsonrpc';
/**
* A position within a QL file.
*/
export interface Position {
/**
* The one-based index of the start line
*/
line: number;
/**
* The one-based offset of the start column within
* the start line in UTF-16 code-units
*/
column: number;
/**
* The one-based index of the end line line
*/
endLine: number;
/**
* The one-based offset of the end column within
* the end line in UTF-16 code-units
*/
endColumn: number;
/**
* The path of the file.
* If the file name is "Compiler Generated" the
* the position is not a real position but
* arises from compiler generated code.
*/
fileName: string;
}
/**
* The way of compiling the query, as a normal query
* or a subset of it. Note that precisely one of the two options should be set.
*/
export interface CompilationTarget {
/**
* Compile as a normal query
*/
query?: Record<string, never>;
/**
* Compile as a quick evaluation
*/
quickEval?: QuickEvalOptions;
}
/**
* Options for quick evaluation
*/
export interface QuickEvalOptions {
quickEvalPos?: Position;
}
/**
* Type for any action that could have progress messages.
*/
export interface WithProgressId<T> {
/**
* The main body
*/
body: T;
/**
* The id used to report progress updates
*/
progressId: number;
}
export interface ProgressMessage {
/**
* The id of the operation that is running
*/
id: number;
/**
* The current step
*/
step: number;
/**
* The maximum step. This *should* be constant for a single job.
*/
maxStep: number;
/**
* The current progress message
*/
message: string;
}
/**
* A notification that the progress has been changed.
*/
export const progress = new rpc.NotificationType<ProgressMessage, void>('ql/progressUpdated');

18
extensions/ql-vscode/src/query-history.ts
View File

@@ -26,7 +26,6 @@ import {
} from './helpers';
import { logger } from './logging';
import { URLSearchParams } from 'url';
import { QueryServerClient } from './queryserver-client';
import { DisposableObject } from './pure/disposable-object';
import { commandRunner } from './commandRunner';
import { ONE_HOUR_IN_MS, TWO_HOURS_IN_MS } from './pure/time';
@@ -48,7 +47,8 @@ import { WebviewReveal } from './interface-utils';
import { EvalLogViewer } from './eval-log-viewer';
import EvalLogTreeBuilder from './eval-log-tree-builder';
import { EvalLogData, parseViewerData } from './pure/log-summary-parser';
import { QueryWithResults } from './run-queries';
import { QueryWithResults } from './run-queries-shared';
import { QueryRunner } from './queryRunner';
/**
* query-history.ts
@@ -329,7 +329,7 @@ export class QueryHistoryManager extends DisposableObject {
readonly onDidCompleteQuery = this._onDidCompleteQuery.event;
constructor(
private readonly qs: QueryServerClient,
private readonly qs: QueryRunner,
private readonly dbm: DatabaseManager,
private readonly localQueriesResultsView: ResultsView,
private readonly remoteQueriesManager: RemoteQueriesManager,
@@ -795,7 +795,7 @@ export class QueryHistoryManager extends DisposableObject {
throw new Error('Please select a local query.');
}
if (!finalSingleItem.completedQuery?.didRunSuccessfully) {
if (!finalSingleItem.completedQuery?.sucessful) {
throw new Error('Please select a query that has completed successfully.');
}
@@ -1076,7 +1076,7 @@ export class QueryHistoryManager extends DisposableObject {
void this.tryOpenExternalFile(query.csvPath);
return;
}
if (await query.exportCsvResults(this.qs, query.csvPath)) {
if (await query.exportCsvResults(this.qs.cliServer, query.csvPath)) {
void this.tryOpenExternalFile(
query.csvPath
);
@@ -1095,7 +1095,7 @@ export class QueryHistoryManager extends DisposableObject {
}
await this.tryOpenExternalFile(
await finalSingleItem.completedQuery.query.ensureCsvAlerts(this.qs, this.dbm)
await finalSingleItem.completedQuery.query.ensureCsvAlerts(this.qs.cliServer, this.dbm)
);
}
@@ -1111,7 +1111,7 @@ export class QueryHistoryManager extends DisposableObject {
}
await this.tryOpenExternalFile(
await finalSingleItem.completedQuery.query.ensureDilPath(this.qs)
await finalSingleItem.completedQuery.query.ensureDilPath(this.qs.cliServer)
);
}
@@ -1236,7 +1236,7 @@ the file in the file explorer and dragging it into the workspace.`
if (!otherQuery.completedQuery) {
throw new Error('Please select a completed query.');
}
if (!otherQuery.completedQuery.didRunSuccessfully) {
if (!otherQuery.completedQuery.sucessful) {
throw new Error('Please select a successful query.');
}
if (otherQuery.initialInfo.databaseInfo.name !== dbName) {
@@ -1256,7 +1256,7 @@ the file in the file explorer and dragging it into the workspace.`
otherQuery !== singleItem &&
otherQuery.t === 'local' &&
otherQuery.completedQuery &&
otherQuery.completedQuery.didRunSuccessfully &&
otherQuery.completedQuery.sucessful &&
otherQuery.initialInfo.databaseInfo.name === dbName
)
.map((item) => ({

46
extensions/ql-vscode/src/query-results.ts
View File

@@ -1,7 +1,7 @@
import { CancellationTokenSource, env } from 'vscode';
import { QueryWithResults, QueryEvaluationInfo } from './run-queries';
import * as messages from './pure/messages';
import * as messages from './pure/messages-shared';
import * as legacyMessages from './pure/legacy-messages';
import * as cli from './cli';
import * as fs from 'fs-extra';
import * as path from 'path';
@@ -17,6 +17,8 @@ import {
import { DatabaseInfo } from './pure/interface-types';
import { QueryStatus } from './query-status';
import { RemoteQueryHistoryItem } from './remote-queries/remote-query-history-item';
import { QueryEvaluationInfo, QueryWithResults } from './run-queries-shared';
import { formatLegacyMessage } from './legacy-query-server/run-queries';
/**
* query-results.ts
@@ -44,7 +46,12 @@ export interface InitialQueryInfo {
export class CompletedQueryInfo implements QueryWithResults {
readonly query: QueryEvaluationInfo;
readonly result: messages.EvaluationResult;
readonly message?: string;
readonly sucessful?: boolean;
/**
* The legacy result. This is only set when loading from the query history.
*/
readonly result?: legacyMessages.EvaluationResult;
readonly logFileLocation?: string;
resultCount: number;
@@ -68,16 +75,19 @@ export class CompletedQueryInfo implements QueryWithResults {
interpretedResultsSortState: InterpretedResultsSortState | undefined;
/**
* Note that in the {@link FullQueryInfo.slurp} method, we create a CompletedQueryInfo instance
* Note that in the {@link slurpQueryHistory} method, we create a CompletedQueryInfo instance
* by explicitly setting the prototype in order to avoid calling this constructor.
*/
constructor(
evaluation: QueryWithResults,
) {
this.query = evaluation.query;
this.result = evaluation.result;
this.logFileLocation = evaluation.logFileLocation;
if (evaluation.result) {
this.result = evaluation.result;
}
this.message = evaluation.message;
this.sucessful = evaluation.sucessful;
// Use the dispose method from the evaluation.
// The dispose will clean up any additional log locations that this
// query may have created.
@@ -92,18 +102,12 @@ export class CompletedQueryInfo implements QueryWithResults {
}
get statusString(): string {
switch (this.result.resultType) {
case messages.QueryResultType.CANCELLATION:
return `cancelled after ${Math.round(this.result.evaluationTime / 1000)} seconds`;
case messages.QueryResultType.OOM:
return 'out of memory';
case messages.QueryResultType.SUCCESS:
return `finished in ${Math.round(this.result.evaluationTime / 1000)} seconds`;
case messages.QueryResultType.TIMEOUT:
return `timed out after ${Math.round(this.result.evaluationTime / 1000)} seconds`;
case messages.QueryResultType.OTHER_ERROR:
default:
return this.result.message ? `failed: ${this.result.message}` : 'failed';
if (this.message) {
return this.message;
} else if (this.result) {
return formatLegacyMessage(this.result);
} else {
throw new Error('No status available');
}
}
@@ -115,10 +119,6 @@ export class CompletedQueryInfo implements QueryWithResults {
|| this.query.resultsPaths.resultsPath;
}
get didRunSuccessfully(): boolean {
return this.result.resultType === messages.QueryResultType.SUCCESS;
}
async updateSortState(
server: cli.CodeQLCliServer,
resultSetName: string,
@@ -302,7 +302,7 @@ export class LocalQueryInfo {
return QueryStatus.Failed;
} else if (!this.completedQuery) {
return QueryStatus.InProgress;
} else if (this.completedQuery.didRunSuccessfully) {
} else if (this.completedQuery.sucessful) {
return QueryStatus.Completed;
} else {
return QueryStatus.Failed;

2
extensions/ql-vscode/src/query-serialization.ts
View File

@@ -5,7 +5,7 @@ import { showAndLogErrorMessage } from './helpers';
import { asyncFilter, getErrorMessage, getErrorStack } from './pure/helpers-pure';
import { CompletedQueryInfo, LocalQueryInfo, QueryHistoryInfo } from './query-results';
import { QueryStatus } from './query-status';
import { QueryEvaluationInfo } from './run-queries';
import { QueryEvaluationInfo } from './run-queries-shared';
export async function slurpQueryHistory(fsPath: string): Promise<QueryHistoryInfo[]> {
try {

48
extensions/ql-vscode/src/queryRunner.ts Normal file
View File

@@ -0,0 +1,48 @@
import { CancellationToken } from 'vscode';
import { CodeQLCliServer } from './cli';
import { ProgressCallback } from './commandRunner';
import { DatabaseItem } from './databases';
import { InitialQueryInfo, LocalQueryInfo } from './query-results';
import { QueryWithResults } from './run-queries-shared';
export abstract class QueryRunner {
abstract restartQueryServer(progress: ProgressCallback, token: CancellationToken): Promise<void>;
abstract cliServer: CodeQLCliServer;
abstract onStart(arg0: (progress: ProgressCallback, token: CancellationToken) => Promise<void>): void;
abstract clearCacheInDatabase(
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken): Promise<void>;
abstract compileAndRunQueryAgainstDatabase(
dbItem: DatabaseItem,
initialInfo: InitialQueryInfo,
queryStorageDir: string,
progress: ProgressCallback,
token: CancellationToken,
templates?: Record<string, string>,
queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
): Promise<QueryWithResults>;
abstract deregisterDatabase(
progress: ProgressCallback,
token: CancellationToken,
dbItem: DatabaseItem,
): Promise<void>;
abstract registerDatabase(
progress: ProgressCallback,
token: CancellationToken,
dbItem: DatabaseItem,
): Promise<void>;
abstract upgradeDatabaseExplicit(
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<void>
}

7
extensions/ql-vscode/src/remote-queries/gh-api/gh-api-client.ts
View File

@@ -6,6 +6,7 @@ import {
VariantAnalysisRepoTask,
VariantAnalysisSubmissionRequest
} from './variant-analysis';
import { Repository } from './repository';
export async function submitVariantAnalysis(
credentials: Credentials,
@@ -73,13 +74,13 @@ export async function getVariantAnalysisRepo(
return response.data;
}
export async function getRepositoryIdFromNwo(
export async function getRepositoryFromNwo(
credentials: Credentials,
owner: string,
repo: string
): Promise<number> {
): Promise<Repository> {
const octokit = await credentials.getOctokit();
const response = await octokit.rest.repos.get({ owner, repo });
return response.data.id;
return response.data as Repository;
}

2
extensions/ql-vscode/src/remote-queries/remote-query-submission-result.ts
View File

@@ -1,6 +1,8 @@
import { RemoteQuery } from './remote-query';
import { VariantAnalysis } from './shared/variant-analysis';
export interface RemoteQuerySubmissionResult {
queryDirPath?: string;
query?: RemoteQuery;
variantAnalysis?: VariantAnalysis;
}

203
extensions/ql-vscode/src/remote-queries/run-remote-query.ts
View File

@@ -17,14 +17,17 @@ import {
import { Credentials } from '../authentication';
import * as cli from '../cli';
import { logger } from '../logging';
import { getActionBranch, getRemoteControllerRepo, setRemoteControllerRepo } from '../config';
import { getActionBranch, getRemoteControllerRepo, isVariantAnalysisLiveResultsEnabled, setRemoteControllerRepo } from '../config';
import { ProgressCallback, UserCancellationException } from '../commandRunner';
import { OctokitResponse } from '@octokit/types/dist-types';
import { OctokitResponse, RequestError } from '@octokit/types/dist-types';
import { RemoteQuery } from './remote-query';
import { RemoteQuerySubmissionResult } from './remote-query-submission-result';
import { QueryMetadata } from '../pure/interface-types';
import { getErrorMessage, REPO_REGEX } from '../pure/helpers-pure';
import * as ghApiClient from './gh-api/gh-api-client';
import { getRepositorySelection, isValidSelection, RepositorySelection } from './repository-selection';
import { parseVariantAnalysisQueryLanguage, VariantAnalysis, VariantAnalysisStatus, VariantAnalysisSubmission } from './shared/variant-analysis';
import { Repository } from './shared/repository';
export interface QlPack {
name: string;
@@ -210,31 +213,7 @@ export async function runRemoteQuery(
message: 'Determining controller repo'
});
// Get the controller repo from the config, if it exists.
// If it doesn't exist, prompt the user to enter it, and save that value to the config.
let controllerRepo: string | undefined;
controllerRepo = getRemoteControllerRepo();
if (!controllerRepo || !REPO_REGEX.test(controllerRepo)) {
void logger.log(controllerRepo ? 'Invalid controller repository name.' : 'No controller repository defined.');
controllerRepo = await window.showInputBox({
title: 'Controller repository in which to run the GitHub Actions workflow for this variant analysis',
placeHolder: '<owner>/<repo>',
prompt: 'Enter the name of a GitHub repository in the format <owner>/<repo>',
ignoreFocusOut: true,
});
if (!controllerRepo) {
void showAndLogErrorMessage('No controller repository entered.');
return;
} else if (!REPO_REGEX.test(controllerRepo)) { // Check if user entered invalid input
void showAndLogErrorMessage('Invalid repository format. Must be a valid GitHub repository in the format <owner>/<repo>.');
return;
}
void logger.log(`Setting the controller repository as: ${controllerRepo}`);
await setRemoteControllerRepo(controllerRepo);
}
void logger.log(`Using controller repository: ${controllerRepo}`);
const [owner, repo] = controllerRepo.split('/');
const controllerRepo = await getControllerRepo(credentials);
progress({
maxStep: 4,
@@ -259,31 +238,84 @@ export async function runRemoteQuery(
});
const actionBranch = getActionBranch();
const apiResponse = await runRemoteQueriesApiRequest(credentials, actionBranch, language, repoSelection, owner, repo, base64Pack, dryRun);
const queryStartTime = Date.now();
const queryMetadata = await tryGetQueryMetadata(cliServer, queryFile);
if (dryRun) {
return { queryDirPath: remoteQueryDir.path };
} else {
if (!apiResponse) {
return;
if (isVariantAnalysisLiveResultsEnabled()) {
const queryName = getQueryName(queryMetadata, queryFile);
const variantAnalysisLanguage = parseVariantAnalysisQueryLanguage(language);
if (variantAnalysisLanguage === undefined) {
throw new UserCancellationException(`Found unsupported language: ${language}`);
}
const workflowRunId = apiResponse.workflow_run_id;
const repositoryCount = apiResponse.repositories_queried.length;
const remoteQuery = await buildRemoteQueryEntity(
queryFile,
queryMetadata,
owner,
repo,
queryStartTime,
workflowRunId,
language,
repositoryCount);
const variantAnalysisSubmission: VariantAnalysisSubmission = {
startTime: queryStartTime,
actionRepoRef: actionBranch,
controllerRepoId: controllerRepo.id,
query: {
name: queryName,
filePath: queryFile,
pack: base64Pack,
language: variantAnalysisLanguage,
},
databases: {
repositories: repoSelection.repositories,
repositoryLists: repoSelection.repositoryLists,
repositoryOwners: repoSelection.owners
}
};
// don't return the path because it has been deleted
return { query: remoteQuery };
const variantAnalysisResponse = await ghApiClient.submitVariantAnalysis(
credentials,
variantAnalysisSubmission
);
const variantAnalysis: VariantAnalysis = {
id: variantAnalysisResponse.id,
controllerRepoId: variantAnalysisResponse.controller_repo.id,
query: {
name: variantAnalysisSubmission.query.name,
filePath: variantAnalysisSubmission.query.filePath,
language: variantAnalysisSubmission.query.language,
},
databases: {
repositories: variantAnalysisSubmission.databases.repositories,
repositoryLists: variantAnalysisSubmission.databases.repositoryLists,
repositoryOwners: variantAnalysisSubmission.databases.repositoryOwners,
},
status: VariantAnalysisStatus.InProgress,
};
// TODO: Remove once we have a proper notification
void showAndLogInformationMessage('Variant analysis submitted for processing');
void logger.log(`Variant analysis:\n${JSON.stringify(variantAnalysis, null, 2)}`);
return { variantAnalysis };
} else {
const apiResponse = await runRemoteQueriesApiRequest(credentials, actionBranch, language, repoSelection, controllerRepo, base64Pack, dryRun);
if (dryRun) {
return { queryDirPath: remoteQueryDir.path };
} else {
if (!apiResponse) {
return;
}
const workflowRunId = apiResponse.workflow_run_id;
const repositoryCount = apiResponse.repositories_queried.length;
const remoteQuery = await buildRemoteQueryEntity(
queryFile,
queryMetadata,
controllerRepo,
queryStartTime,
workflowRunId,
language,
repositoryCount);
// don't return the path because it has been deleted
return { query: remoteQuery };
}
}
} finally {
@@ -301,8 +333,7 @@ async function runRemoteQueriesApiRequest(
ref: string,
language: string,
repoSelection: RepositorySelection,
owner: string,
repo: string,
controllerRepo: Repository,
queryPackBase64: string,
dryRun = false
): Promise<void | QueriesResponse> {
@@ -318,8 +349,7 @@ async function runRemoteQueriesApiRequest(
if (dryRun) {
void showAndLogInformationMessage('[DRY RUN] Would have sent request. See extension log for the payload.');
void logger.log(JSON.stringify({
owner,
repo,
controllerRepo,
data: {
...data,
queryPackBase64: queryPackBase64.substring(0, 100) + '... ' + queryPackBase64.length + ' bytes'
@@ -331,14 +361,13 @@ async function runRemoteQueriesApiRequest(
try {
const octokit = await credentials.getOctokit();
const response: OctokitResponse<QueriesResponse, number> = await octokit.request(
'POST /repos/:owner/:repo/code-scanning/codeql/queries',
'POST /repos/:controllerRepo/code-scanning/codeql/queries',
{
owner,
repo,
controllerRepo: controllerRepo.fullName,
data
}
);
const { popupMessage, logMessage } = parseResponse(owner, repo, response.data);
const { popupMessage, logMessage } = parseResponse(controllerRepo, response.data);
void showAndLogInformationMessage(popupMessage, { fullMessage: logMessage });
return response.data;
} catch (error: any) {
@@ -354,14 +383,14 @@ const eol = os.EOL;
const eol2 = os.EOL + os.EOL;
// exported for testing only
export function parseResponse(owner: string, repo: string, response: QueriesResponse) {
export function parseResponse(controllerRepo: Repository, response: QueriesResponse) {
const repositoriesQueried = response.repositories_queried;
const repositoryCount = repositoriesQueried.length;
const popupMessage = `Successfully scheduled runs on ${pluralize(repositoryCount, 'repository', 'repositories')}. [Click here to see the progress](https://github.com/${owner}/${repo}/actions/runs/${response.workflow_run_id}).`
const popupMessage = `Successfully scheduled runs on ${pluralize(repositoryCount, 'repository', 'repositories')}. [Click here to see the progress](https://github.com/${controllerRepo.fullName}/actions/runs/${response.workflow_run_id}).`
+ (response.errors ? `${eol2}Some repositories could not be scheduled. See extension log for details.` : '');
let logMessage = `Successfully scheduled runs on ${pluralize(repositoryCount, 'repository', 'repositories')}. See https://github.com/${owner}/${repo}/actions/runs/${response.workflow_run_id}.`;
let logMessage = `Successfully scheduled runs on ${pluralize(repositoryCount, 'repository', 'repositories')}. See https://github.com/${controllerRepo.fullName}/actions/runs/${response.workflow_run_id}.`;
logMessage += `${eol2}Repositories queried:${eol}${repositoriesQueried.join(', ')}`;
if (response.errors) {
const { invalid_repositories, repositories_without_database, private_repositories, cutoff_repositories, cutoff_repositories_count } = response.errors;
@@ -425,17 +454,15 @@ async function ensureNameAndSuite(queryPackDir: string, packRelativePath: string
async function buildRemoteQueryEntity(
queryFilePath: string,
queryMetadata: QueryMetadata | undefined,
controllerRepoOwner: string,
controllerRepoName: string,
controllerRepo: Repository,
queryStartTime: number,
workflowRunId: number,
language: string,
repositoryCount: number
): Promise<RemoteQuery> {
// The query name is either the name as specified in the query metadata, or the file name.
const queryName = queryMetadata?.name ?? path.basename(queryFilePath);
const queryName = getQueryName(queryMetadata, queryFilePath);
const queryText = await fs.readFile(queryFilePath, 'utf8');
const [owner, name] = controllerRepo.fullName.split('/');
return {
queryName,
@@ -443,11 +470,59 @@ async function buildRemoteQueryEntity(
queryText,
language,
controllerRepository: {
owner: controllerRepoOwner,
name: controllerRepoName,
owner,
name,
},
executionStartTime: queryStartTime,
actionsWorkflowRunId: workflowRunId,
repositoryCount,
};
}
function getQueryName(queryMetadata: QueryMetadata | undefined, queryFilePath: string): string {
// The query name is either the name as specified in the query metadata, or the file name.
return queryMetadata?.name ?? path.basename(queryFilePath);
}
async function getControllerRepo(credentials: Credentials): Promise<Repository> {
// Get the controller repo from the config, if it exists.
// If it doesn't exist, prompt the user to enter it, and save that value to the config.
let controllerRepoNwo: string | undefined;
controllerRepoNwo = getRemoteControllerRepo();
if (!controllerRepoNwo || !REPO_REGEX.test(controllerRepoNwo)) {
void logger.log(controllerRepoNwo ? 'Invalid controller repository name.' : 'No controller repository defined.');
controllerRepoNwo = await window.showInputBox({
title: 'Controller repository in which to run the GitHub Actions workflow for this variant analysis',
placeHolder: '<owner>/<repo>',
prompt: 'Enter the name of a GitHub repository in the format <owner>/<repo>',
ignoreFocusOut: true,
});
if (!controllerRepoNwo) {
throw new UserCancellationException('No controller repository entered.');
} else if (!REPO_REGEX.test(controllerRepoNwo)) { // Check if user entered invalid input
throw new UserCancellationException('Invalid repository format. Must be a valid GitHub repository in the format <owner>/<repo>.');
}
void logger.log(`Setting the controller repository as: ${controllerRepoNwo}`);
await setRemoteControllerRepo(controllerRepoNwo);
}
void logger.log(`Using controller repository: ${controllerRepoNwo}`);
const [owner, repo] = controllerRepoNwo.split('/');
try {
const controllerRepo = await ghApiClient.getRepositoryFromNwo(credentials, owner, repo);
void logger.log(`Controller repository ID: ${controllerRepo.id}`);
return {
id: controllerRepo.id,
fullName: controllerRepo.full_name,
private: controllerRepo.private,
};
} catch (e: any) {
if ((e as RequestError).status === 404) {
throw new Error(`Controller repository "${owner}/${repo}" not found`);
} else {
throw new Error(`Error getting controller repository "${owner}/${repo}": ${e.message}`);
}
}
}

4
extensions/ql-vscode/src/remote-queries/shared/variant-analysis.ts
View File

@@ -30,6 +30,10 @@ export enum VariantAnalysisQueryLanguage {
Ruby = 'ruby'
}
export function parseVariantAnalysisQueryLanguage(language: string): VariantAnalysisQueryLanguage | undefined {
return Object.values(VariantAnalysisQueryLanguage).find(x => x === language);
}
export enum VariantAnalysisStatus {
InProgress = 'inProgress',
Succeeded = 'succeeded',

584
extensions/ql-vscode/src/run-queries-shared.ts Normal file
View File

@@ -0,0 +1,584 @@
import * as messages from './pure/messages-shared';
import * as legacyMessages from './pure/legacy-messages';
import { DatabaseInfo, QueryMetadata } from './pure/interface-types';
import * as path from 'path';
import { createTimestampFile, showAndLogWarningMessage } from './helpers';
import {
ConfigurationTarget,
Range,
TextDocument,
TextEditor,
Uri,
window
} from 'vscode';
import * as config from './config';
import { UserCancellationException } from './commandRunner';
import * as fs from 'fs-extra';
import { ensureMetadataIsComplete, InitialQueryInfo, LocalQueryInfo } from './query-results';
import { isQuickQueryPath } from './quick-query';
import { nanoid } from 'nanoid';
import { CodeQLCliServer } from './cli';
import { SELECT_QUERY_NAME } from './contextual/locationFinder';
import { DatabaseManager } from './databases';
import { DecodedBqrsChunk } from './pure/bqrs-cli-types';
import { logger, Logger } from './logging';
import { generateSummarySymbolsFile } from './log-insights/summary-parser';
import { asError } from './pure/helpers-pure';
/**
* run-queries.ts
* --------------
*
* Compiling and running QL queries.
*/
export function findQueryLogFile(resultPath: string): string {
return path.join(resultPath, 'query.log');
}
function findQueryEvalLogFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.jsonl');
}
function findQueryEvalLogSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary');
}
function findJsonQueryEvalLogSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary.jsonl');
}
function findQueryEvalLogSummarySymbolsFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary.symbols.json');
}
function findQueryEvalLogEndSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log-end.summary');
}
export class QueryEvaluationInfo {
/**
* Note that in the {@link slurpQueryHistory} method, we create a QueryEvaluationInfo instance
* by explicitly setting the prototype in order to avoid calling this constructor.
*/
constructor(
public readonly querySaveDir: string,
public readonly dbItemPath: string,
private readonly databaseHasMetadataFile: boolean,
public readonly quickEvalPosition?: messages.Position,
public readonly metadata?: QueryMetadata,
) {
/**/
}
get dilPath() {
return path.join(this.querySaveDir, 'results.dil');
}
/**
* Get the path that the compiled query is if it exists. Note that it only exists when using the legacy query server.
*/
get compileQueryPath() {
return path.join(this.querySaveDir, 'compiledQuery.qlo');
}
get csvPath() {
return path.join(this.querySaveDir, 'results.csv');
}
get logPath() {
return findQueryLogFile(this.querySaveDir);
}
get evalLogPath() {
return findQueryEvalLogFile(this.querySaveDir);
}
get evalLogSummaryPath() {
return findQueryEvalLogSummaryFile(this.querySaveDir);
}
get jsonEvalLogSummaryPath() {
return findJsonQueryEvalLogSummaryFile(this.querySaveDir);
}
get evalLogSummarySymbolsPath() {
return findQueryEvalLogSummarySymbolsFile(this.querySaveDir);
}
get evalLogEndSummaryPath() {
return findQueryEvalLogEndSummaryFile(this.querySaveDir);
}
get resultsPaths() {
return {
resultsPath: path.join(this.querySaveDir, 'results.bqrs'),
interpretedResultsPath: path.join(this.querySaveDir,
this.metadata?.kind === 'graph'
? 'graphResults'
: 'interpretedResults.sarif'
),
};
}
getSortedResultSetPath(resultSetName: string) {
return path.join(this.querySaveDir, `sortedResults-${resultSetName}.bqrs`);
}
/**
* Creates a file in the query directory that indicates when this query was created.
* This is important for keeping track of when queries should be removed.
*/
async createTimestampFile() {
await createTimestampFile(this.querySaveDir);
}
/**
* Holds if this query can in principle produce interpreted results.
*/
canHaveInterpretedResults(): boolean {
if (!this.databaseHasMetadataFile) {
void logger.log('Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.');
return false;
}
const kind = this.metadata?.kind;
const hasKind = !!kind;
if (!hasKind) {
void logger.log('Cannot produce interpreted results since the query does not have @kind metadata.');
return false;
}
// Graph queries only return interpreted results if we are in canary mode.
if (kind === 'graph') {
return config.isCanary();
}
// table is the default query kind. It does not produce interpreted results.
// any query kind that is not table can, in principle, produce interpreted results.
return kind !== 'table';
}
/**
* Holds if this query actually has produced interpreted results.
*/
async hasInterpretedResults(): Promise<boolean> {
return fs.pathExists(this.resultsPaths.interpretedResultsPath);
}
/**
* Holds if this query already has DIL produced
*/
async hasDil(): Promise<boolean> {
return fs.pathExists(this.dilPath);
}
/**
* Holds if this query already has CSV results produced
*/
async hasCsv(): Promise<boolean> {
return fs.pathExists(this.csvPath);
}
/**
* Returns the path to the DIL file produced by this query. If the query has not yet produced DIL,
* this will return first create the DIL file and then return the path to the DIL file.
*/
async ensureDilPath(cliServer: CodeQLCliServer): Promise<string> {
if (await this.hasDil()) {
return this.dilPath;
}
const compiledQuery = path.join(this.querySaveDir, 'compiledQuery.qlo');
if (!(await fs.pathExists(compiledQuery))) {
throw new Error(
`Cannot create DIL because compiled query is missing. ${compiledQuery}`
);
}
await cliServer.generateDil(compiledQuery, this.dilPath);
return this.dilPath;
}
/**
* Holds if this query already has a completed structured evaluator log
*/
async hasEvalLog(): Promise<boolean> {
return fs.pathExists(this.evalLogPath);
}
async addQueryLogs(queryInfo: LocalQueryInfo, cliServer: CodeQLCliServer, logger: Logger) {
queryInfo.evalLogLocation = this.evalLogPath;
queryInfo.evalLogSummaryLocation = await this.generateHumanReadableLogSummary(cliServer);
void this.logEndSummary(queryInfo.evalLogSummaryLocation, logger); // Logged asynchrnously
if (config.isCanary()) { // Generate JSON summary for viewer.
await cliServer.generateJsonLogSummary(this.evalLogPath, this.jsonEvalLogSummaryPath);
queryInfo.jsonEvalLogSummaryLocation = this.jsonEvalLogSummaryPath;
await generateSummarySymbolsFile(this.evalLogSummaryPath, this.evalLogSummarySymbolsPath);
queryInfo.evalLogSummarySymbolsLocation = this.evalLogSummarySymbolsPath;
}
}
/**
* Calls the appropriate CLI command to generate a human-readable log summary.
* @param qs The query server client.
* @returns The path to the log summary, or `undefined` if the summary could not be generated. */
private async generateHumanReadableLogSummary(cliServer: CodeQLCliServer): Promise<string | undefined> {
try {
await cliServer.generateLogSummary(this.evalLogPath, this.evalLogSummaryPath, this.evalLogEndSummaryPath);
return this.evalLogSummaryPath;
} catch (e) {
const err = asError(e);
void showAndLogWarningMessage(`Failed to generate human-readable structured evaluator log summary. Reason: ${err.message}`);
return undefined;
}
}
/**
* Logs the end summary to the Output window and log file.
* @param logSummaryPath Path to the human-readable log summary
* @param qs The query server client.
*/
private async logEndSummary(logSummaryPath: string | undefined, logger: Logger): Promise<void> {
if (logSummaryPath === undefined) {
// Failed to generate the log, so we don't expect an end summary either.
return;
}
try {
const endSummaryContent = await fs.readFile(this.evalLogEndSummaryPath, 'utf-8');
void logger.log(' --- Evaluator Log Summary --- ', { additionalLogLocation: this.logPath });
void logger.log(endSummaryContent, { additionalLogLocation: this.logPath });
} catch (e) {
void showAndLogWarningMessage(`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`);
}
}
/**
* Creates the CSV file containing the results of this query. This will only be called if the query
* does not have interpreted results and the CSV file does not already exist.
*
* @return Promise<true> if the operation creates the file. Promise<false> if the operation does
* not create the file.
*
* @throws Error if the operation fails.
*/
async exportCsvResults(cliServer: CodeQLCliServer, csvPath: string): Promise<boolean> {
const resultSet = await this.chooseResultSet(cliServer);
if (!resultSet) {
void showAndLogWarningMessage('Query has no result set.');
return false;
}
let stopDecoding = false;
const out = fs.createWriteStream(csvPath);
const promise: Promise<boolean> = new Promise((resolve, reject) => {
out.on('finish', () => resolve(true));
out.on('error', () => {
if (!stopDecoding) {
stopDecoding = true;
reject(new Error(`Failed to write CSV results to ${csvPath}`));
}
});
});
let nextOffset: number | undefined = 0;
do {
const chunk: DecodedBqrsChunk = await cliServer.bqrsDecode(this.resultsPaths.resultsPath, resultSet, {
pageSize: 100,
offset: nextOffset,
});
chunk.tuples.forEach((tuple) => {
out.write(tuple.map((v, i) =>
chunk.columns[i].kind === 'String'
? `"${typeof v === 'string' ? v.replaceAll('"', '""') : v}"`
: v
).join(',') + '\n');
});
nextOffset = chunk.next;
} while (nextOffset && !stopDecoding);
out.end();
return promise;
}
/**
* Choose the name of the result set to run. If the `#select` set exists, use that. Otherwise,
* arbitrarily choose the first set. Most of the time, this will be correct.
*
* If the query has no result sets, then return undefined.
*/
async chooseResultSet(cliServer: CodeQLCliServer) {
const resultSets = (await cliServer.bqrsInfo(this.resultsPaths.resultsPath, 0))['result-sets'];
if (!resultSets.length) {
return undefined;
}
if (resultSets.find(r => r.name === SELECT_QUERY_NAME)) {
return SELECT_QUERY_NAME;
}
return resultSets[0].name;
}
/**
* Returns the path to the CSV alerts interpretation of this query results. If CSV results have
* not yet been produced, this will return first create the CSV results and then return the path.
*
* This method only works for queries with interpreted results.
*/
async ensureCsvAlerts(cliServer: CodeQLCliServer, dbm: DatabaseManager): Promise<string> {
if (await this.hasCsv()) {
return this.csvPath;
}
const dbItem = dbm.findDatabaseItem(Uri.file(this.dbItemPath));
if (!dbItem) {
throw new Error(`Cannot produce CSV results because database is missing. ${this.dbItemPath}`);
}
let sourceInfo;
if (dbItem.sourceArchive !== undefined) {
sourceInfo = {
sourceArchive: dbItem.sourceArchive.fsPath,
sourceLocationPrefix: await dbItem.getSourceLocationPrefix(
cliServer
),
};
}
await cliServer.generateResultsCsv(ensureMetadataIsComplete(this.metadata), this.resultsPaths.resultsPath, this.csvPath, sourceInfo);
return this.csvPath;
}
/**
* Cleans this query's results directory.
*/
async deleteQuery(): Promise<void> {
await fs.remove(this.querySaveDir);
}
}
export interface QueryWithResults {
readonly query: QueryEvaluationInfo;
readonly logFileLocation?: string;
readonly dispose: () => void;
readonly sucessful?: boolean;
readonly message?: string;
readonly result?: legacyMessages.EvaluationResult
}
/**
* Information about which query will be to be run. `quickEvalPosition` and `quickEvalText`
* is only filled in if the query is a quick query.
*/
interface SelectedQuery {
queryPath: string;
quickEvalPosition?: messages.Position;
quickEvalText?: string;
}
/**
* Determines which QL file to run during an invocation of `Run Query` or `Quick Evaluation`, as follows:
* - If the command was called by clicking on a file, then use that file.
* - Otherwise, use the file open in the current editor.
* - In either case, prompt the user to save the file if it is open with unsaved changes.
* - For `Quick Evaluation`, ensure the selected file is also the one open in the editor,
* and use the selected region.
* @param selectedResourceUri The selected resource when the command was run.
* @param quickEval Whether the command being run is `Quick Evaluation`.
*/
export async function determineSelectedQuery(
selectedResourceUri: Uri | undefined,
quickEval: boolean,
range?: Range
): Promise<SelectedQuery> {
const editor = window.activeTextEditor;
// Choose which QL file to use.
let queryUri: Uri;
if (selectedResourceUri) {
// A resource was passed to the command handler, so use it.
queryUri = selectedResourceUri;
} else {
// No resource was passed to the command handler, so obtain it from the active editor.
// This usually happens when the command is called from the Command Palette.
if (editor === undefined) {
throw new Error('No query was selected. Please select a query and try again.');
} else {
queryUri = editor.document.uri;
}
}
if (queryUri.scheme !== 'file') {
throw new Error('Can only run queries that are on disk.');
}
const queryPath = queryUri.fsPath;
if (quickEval) {
if (!(queryPath.endsWith('.ql') || queryPath.endsWith('.qll'))) {
throw new Error('The selected resource is not a CodeQL file; It should have the extension ".ql" or ".qll".');
}
} else {
if (!(queryPath.endsWith('.ql'))) {
throw new Error('The selected resource is not a CodeQL query file; It should have the extension ".ql".');
}
}
// Whether we chose the file from the active editor or from a context menu,
// if the same file is open with unsaved changes in the active editor,
// then prompt the user to save it first.
if (editor !== undefined && editor.document.uri.fsPath === queryPath) {
if (await promptUserToSaveChanges(editor.document)) {
await editor.document.save();
}
}
let quickEvalPosition: messages.Position | undefined = undefined;
let quickEvalText: string | undefined = undefined;
if (quickEval) {
if (editor == undefined) {
throw new Error('Can\'t run quick evaluation without an active editor.');
}
if (editor.document.fileName !== queryPath) {
// For Quick Evaluation we expect these to be the same.
// Report an error if we end up in this (hopefully unlikely) situation.
throw new Error('The selected resource for quick evaluation should match the active editor.');
}
quickEvalPosition = await getSelectedPosition(editor, range);
if (!editor.selection?.isEmpty) {
quickEvalText = editor.document.getText(editor.selection);
} else {
// capture the entire line if the user didn't select anything
const line = editor.document.lineAt(editor.selection.active.line);
quickEvalText = line.text.trim();
}
}
return { queryPath, quickEvalPosition, quickEvalText };
}
/** Gets the selected position within the given editor. */
async function getSelectedPosition(editor: TextEditor, range?: Range): Promise<messages.Position> {
const selectedRange = range || editor.selection;
const pos = selectedRange.start;
const posEnd = selectedRange.end;
// Convert from 0-based to 1-based line and column numbers.
return {
fileName: await convertToQlPath(editor.document.fileName),
line: pos.line + 1,
column: pos.character + 1,
endLine: posEnd.line + 1,
endColumn: posEnd.character + 1
};
}
/**
* Prompts the user to save `document` if it has unsaved changes.
*
* @param document The document to save.
*
* @returns true if we should save changes and false if we should continue without saving changes.
* @throws UserCancellationException if we should abort whatever operation triggered this prompt
*/
async function promptUserToSaveChanges(document: TextDocument): Promise<boolean> {
if (document.isDirty) {
if (config.AUTOSAVE_SETTING.getValue()) {
return true;
}
else {
const yesItem = { title: 'Yes', isCloseAffordance: false };
const alwaysItem = { title: 'Always Save', isCloseAffordance: false };
const noItem = { title: 'No (run version on disk)', isCloseAffordance: false };
const cancelItem = { title: 'Cancel', isCloseAffordance: true };
const message = 'Query file has unsaved changes. Save now?';
const chosenItem = await window.showInformationMessage(
message,
{ modal: true },
yesItem, alwaysItem, noItem, cancelItem
);
if (chosenItem === alwaysItem) {
await config.AUTOSAVE_SETTING.updateValue(true, ConfigurationTarget.Workspace);
return true;
}
if (chosenItem === yesItem) {
return true;
}
if (chosenItem === cancelItem) {
throw new UserCancellationException('Query run cancelled.', true);
}
}
}
return false;
}
/**
* @param filePath This needs to be equivalent to Java's `Path.toRealPath(NO_FOLLOW_LINKS)`
*/
async function convertToQlPath(filePath: string): Promise<string> {
if (process.platform === 'win32') {
if (path.parse(filePath).root === filePath) {
// Java assumes uppercase drive letters are canonical.
return filePath.toUpperCase();
} else {
const dir = await convertToQlPath(path.dirname(filePath));
const fileName = path.basename(filePath);
const fileNames = await fs.readdir(dir);
for (const name of fileNames) {
// Leave the locale argument empty so that the default OS locale is used.
// We do this because this operation works on filesystem entities, which
// use the os locale, regardless of the locale of the running VS Code instance.
if (fileName.localeCompare(name, undefined, { sensitivity: 'accent' }) === 0) {
return path.join(dir, name);
}
}
}
throw new Error('Can\'t convert path to form suitable for QL:' + filePath);
} else {
return filePath;
}
}
/**
* Determines the initial information for a query. This is everything of interest
* we know about this query that is available before it is run.
*
* @param selectedQueryUri The Uri of the document containing the query to be run.
* @param databaseInfo The database to run the query against.
* @param isQuickEval true if this is a quick evaluation.
* @param range the selection range of the query to be run. Only used if isQuickEval is true.
* @returns The initial information for the query to be run.
*/
export async function createInitialQueryInfo(
selectedQueryUri: Uri | undefined,
databaseInfo: DatabaseInfo,
isQuickEval: boolean,
range?: Range
): Promise<InitialQueryInfo> {
// Determine which query to run, based on the selection and the active editor.
const { queryPath, quickEvalPosition, quickEvalText } = await determineSelectedQuery(selectedQueryUri, isQuickEval, range);
return {
queryPath,
isQuickEval,
isQuickQuery: isQuickQueryPath(queryPath),
databaseInfo,
id: `${path.basename(queryPath)}-${nanoid()}`,
start: new Date(),
... (isQuickEval ? {
queryText: quickEvalText!, // if this query is quick eval, it must have quick eval text
quickEvalPosition: quickEvalPosition
} : {
queryText: await fs.readFile(queryPath, 'utf8')
})
};
}

997
extensions/ql-vscode/src/run-queries.ts
View File

@@ -1,997 +0,0 @@
import * as crypto from 'crypto';
import * as fs from 'fs-extra';
import * as tmp from 'tmp-promise';
import * as path from 'path';
import { nanoid } from 'nanoid';
import {
CancellationToken,
ConfigurationTarget,
Range,
TextDocument,
TextEditor,
Uri,
window
} from 'vscode';
import { ErrorCodes, ResponseError } from 'vscode-languageclient';
import * as cli from './cli';
import * as config from './config';
import { DatabaseItem, DatabaseManager } from './databases';
import {
createTimestampFile,
getOnDiskWorkspaceFolders,
showAndLogErrorMessage,
showAndLogWarningMessage,
tryGetQueryMetadata,
upgradesTmpDir
} from './helpers';
import { ProgressCallback, UserCancellationException } from './commandRunner';
import { DatabaseInfo, QueryMetadata } from './pure/interface-types';
import { logger } from './logging';
import * as messages from './pure/messages';
import { InitialQueryInfo, LocalQueryInfo } from './query-results';
import * as qsClient from './queryserver-client';
import { isQuickQueryPath } from './quick-query';
import { compileDatabaseUpgradeSequence, upgradeDatabaseExplicit } from './upgrades';
import { ensureMetadataIsComplete } from './query-results';
import { SELECT_QUERY_NAME } from './contextual/locationFinder';
import { DecodedBqrsChunk } from './pure/bqrs-cli-types';
import { asError, getErrorMessage } from './pure/helpers-pure';
import { generateSummarySymbolsFile } from './log-insights/summary-parser';
/**
* run-queries.ts
* --------------
*
* Compiling and running QL queries.
*/
/**
* Information about which query will be to be run. `quickEvalPosition` and `quickEvalText`
* is only filled in if the query is a quick query.
*/
interface SelectedQuery {
queryPath: string;
quickEvalPosition?: messages.Position;
quickEvalText?: string;
}
/**
* A collection of evaluation-time information about a query,
* including the query itself, and where we have decided to put
* temporary files associated with it, such as the compiled query
* output and results.
*/
export class QueryEvaluationInfo {
/**
* Note that in the {@link FullQueryInfo.slurp} method, we create a QueryEvaluationInfo instance
* by explicitly setting the prototype in order to avoid calling this constructor.
*/
constructor(
public readonly querySaveDir: string,
public readonly dbItemPath: string,
private readonly databaseHasMetadataFile: boolean,
public readonly queryDbscheme: string, // the dbscheme file the query expects, based on library path resolution
public readonly quickEvalPosition?: messages.Position,
public readonly metadata?: QueryMetadata,
public readonly templates?: messages.TemplateDefinitions
) {
/**/
}
get dilPath() {
return path.join(this.querySaveDir, 'results.dil');
}
get csvPath() {
return path.join(this.querySaveDir, 'results.csv');
}
get compiledQueryPath() {
return path.join(this.querySaveDir, 'compiledQuery.qlo');
}
get logPath() {
return qsClient.findQueryLogFile(this.querySaveDir);
}
get evalLogPath() {
return qsClient.findQueryEvalLogFile(this.querySaveDir);
}
get evalLogSummaryPath() {
return qsClient.findQueryEvalLogSummaryFile(this.querySaveDir);
}
get jsonEvalLogSummaryPath() {
return qsClient.findJsonQueryEvalLogSummaryFile(this.querySaveDir);
}
get evalLogSummarySymbolsPath() {
return qsClient.findQueryEvalLogSummarySymbolsFile(this.querySaveDir);
}
get evalLogEndSummaryPath() {
return qsClient.findQueryEvalLogEndSummaryFile(this.querySaveDir);
}
get resultsPaths() {
return {
resultsPath: path.join(this.querySaveDir, 'results.bqrs'),
interpretedResultsPath: path.join(this.querySaveDir,
this.metadata?.kind === 'graph'
? 'graphResults'
: 'interpretedResults.sarif'
),
};
}
getSortedResultSetPath(resultSetName: string) {
return path.join(this.querySaveDir, `sortedResults-${resultSetName}.bqrs`);
}
/**
* Creates a file in the query directory that indicates when this query was created.
* This is important for keeping track of when queries should be removed.
*/
async createTimestampFile() {
await createTimestampFile(this.querySaveDir);
}
async run(
qs: qsClient.QueryServerClient,
upgradeQlo: string | undefined,
availableMlModels: cli.MlModelInfo[],
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
queryInfo?: LocalQueryInfo,
): Promise<messages.EvaluationResult> {
if (!dbItem.contents || dbItem.error) {
throw new Error('Can\'t run query on invalid database.');
}
let result: messages.EvaluationResult | null = null;
const callbackId = qs.registerCallback(res => {
result = {
...res,
logFileLocation: this.logPath
};
});
const availableMlModelUris: messages.MlModel[] = availableMlModels.map(model => ({ uri: Uri.file(model.path).toString(true) }));
const queryToRun: messages.QueryToRun = {
resultsPath: this.resultsPaths.resultsPath,
qlo: Uri.file(this.compiledQueryPath).toString(),
compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(),
allowUnknownTemplates: true,
templateValues: this.templates,
availableMlModels: availableMlModelUris,
id: callbackId,
timeoutSecs: qs.config.timeoutSecs,
};
const dataset: messages.Dataset = {
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default'
};
if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
await qs.sendRequest(messages.startLog, {
db: dataset,
logPath: this.evalLogPath,
});
}
const params: messages.EvaluateQueriesParams = {
db: dataset,
evaluateId: callbackId,
queries: [queryToRun],
stopOnError: false,
useSequenceHint: false
};
try {
await qs.sendRequest(messages.runQueries, params, token, progress);
if (qs.config.customLogDirectory) {
void showAndLogWarningMessage(
`Custom log directories are no longer supported. The "codeQL.runningQueries.customLogDirectory" setting is deprecated. Unset the setting to stop seeing this message. Query logs saved to ${this.logPath}.`
);
}
} finally {
qs.unRegisterCallback(callbackId);
if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
await qs.sendRequest(messages.endLog, {
db: dataset,
logPath: this.evalLogPath,
});
if (await this.hasEvalLog()) {
queryInfo.evalLogLocation = this.evalLogPath;
queryInfo.evalLogSummaryLocation = await this.generateHumanReadableLogSummary(qs);
void this.logEndSummary(queryInfo.evalLogSummaryLocation, qs); // Logged asynchrnously
if (config.isCanary()) { // Generate JSON summary for viewer.
await qs.cliServer.generateJsonLogSummary(this.evalLogPath, this.jsonEvalLogSummaryPath);
queryInfo.jsonEvalLogSummaryLocation = this.jsonEvalLogSummaryPath;
await generateSummarySymbolsFile(this.evalLogSummaryPath, this.evalLogSummarySymbolsPath);
queryInfo.evalLogSummarySymbolsLocation = this.evalLogSummarySymbolsPath;
}
} else {
void showAndLogWarningMessage(`Failed to write structured evaluator log to ${this.evalLogPath}.`);
}
}
}
return result || {
evaluationTime: 0,
message: 'No result from server',
queryId: -1,
runId: callbackId,
resultType: messages.QueryResultType.OTHER_ERROR
};
}
async compile(
qs: qsClient.QueryServerClient,
program: messages.QlProgram,
progress: ProgressCallback,
token: CancellationToken,
): Promise<messages.CompilationMessage[]> {
let compiled: messages.CheckQueryResult | undefined;
try {
const target = this.quickEvalPosition ? {
quickEval: { quickEvalPos: this.quickEvalPosition }
} : { query: {} };
const params: messages.CompileQueryParams = {
compilationOptions: {
computeNoLocationUrls: true,
failOnWarnings: false,
fastCompilation: false,
includeDilInQlo: true,
localChecking: false,
noComputeGetUrl: false,
noComputeToString: false,
computeDefaultStrings: true,
emitDebugInfo: true
},
extraOptions: {
timeoutSecs: qs.config.timeoutSecs
},
queryToCheck: program,
resultPath: this.compiledQueryPath,
target,
};
compiled = await qs.sendRequest(messages.compileQuery, params, token, progress);
} finally {
void qs.logger.log(' - - - COMPILATION DONE - - - ', { additionalLogLocation: this.logPath });
}
return (compiled?.messages || []).filter(msg => msg.severity === messages.Severity.ERROR);
}
/**
* Holds if this query can in principle produce interpreted results.
*/
canHaveInterpretedResults(): boolean {
if (!this.databaseHasMetadataFile) {
void logger.log('Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.');
return false;
}
const kind = this.metadata?.kind;
const hasKind = !!kind;
if (!hasKind) {
void logger.log('Cannot produce interpreted results since the query does not have @kind metadata.');
return false;
}
// Graph queries only return interpreted results if we are in canary mode.
if (kind === 'graph') {
return config.isCanary();
}
// table is the default query kind. It does not produce interpreted results.
// any query kind that is not table can, in principle, produce interpreted results.
return kind !== 'table';
}
/**
* Holds if this query actually has produced interpreted results.
*/
async hasInterpretedResults(): Promise<boolean> {
return fs.pathExists(this.resultsPaths.interpretedResultsPath);
}
/**
* Holds if this query already has DIL produced
*/
async hasDil(): Promise<boolean> {
return fs.pathExists(this.dilPath);
}
/**
* Holds if this query already has CSV results produced
*/
async hasCsv(): Promise<boolean> {
return fs.pathExists(this.csvPath);
}
/**
* Returns the path to the DIL file produced by this query. If the query has not yet produced DIL,
* this will return first create the DIL file and then return the path to the DIL file.
*/
async ensureDilPath(qs: qsClient.QueryServerClient): Promise<string> {
if (await this.hasDil()) {
return this.dilPath;
}
if (!(await fs.pathExists(this.compiledQueryPath))) {
throw new Error(
`Cannot create DIL because compiled query is missing. ${this.compiledQueryPath}`
);
}
await qs.cliServer.generateDil(this.compiledQueryPath, this.dilPath);
return this.dilPath;
}
/**
* Holds if this query already has a completed structured evaluator log
*/
async hasEvalLog(): Promise<boolean> {
return fs.pathExists(this.evalLogPath);
}
/**
* Calls the appropriate CLI command to generate a human-readable log summary.
* @param qs The query server client.
* @returns The path to the log summary, or `undefined` if the summary could not be generated.
*/
private async generateHumanReadableLogSummary(qs: qsClient.QueryServerClient): Promise<string | undefined> {
try {
await qs.cliServer.generateLogSummary(this.evalLogPath, this.evalLogSummaryPath, this.evalLogEndSummaryPath);
return this.evalLogSummaryPath;
} catch (e) {
const err = asError(e);
void showAndLogWarningMessage(`Failed to generate human-readable structured evaluator log summary. Reason: ${err.message}`);
return undefined;
}
}
/**
* Logs the end summary to the Output window and log file.
* @param logSummaryPath Path to the human-readable log summary
* @param qs The query server client.
*/
private async logEndSummary(logSummaryPath: string | undefined, qs: qsClient.QueryServerClient): Promise<void> {
if (logSummaryPath === undefined) {
// Failed to generate the log, so we don't expect an end summary either.
return;
}
try {
const endSummaryContent = await fs.readFile(this.evalLogEndSummaryPath, 'utf-8');
void qs.logger.log(' --- Evaluator Log Summary --- ', { additionalLogLocation: this.logPath });
void qs.logger.log(endSummaryContent, { additionalLogLocation: this.logPath });
} catch (e) {
void showAndLogWarningMessage(`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`);
}
}
/**
* Creates the CSV file containing the results of this query. This will only be called if the query
* does not have interpreted results and the CSV file does not already exist.
*
* @return Promise<true> if the operation creates the file. Promise<false> if the operation does
* not create the file.
*
* @throws Error if the operation fails.
*/
async exportCsvResults(qs: qsClient.QueryServerClient, csvPath: string): Promise<boolean> {
const resultSet = await this.chooseResultSet(qs);
if (!resultSet) {
void showAndLogWarningMessage('Query has no result set.');
return false;
}
let stopDecoding = false;
const out = fs.createWriteStream(csvPath);
const promise: Promise<boolean> = new Promise((resolve, reject) => {
out.on('finish', () => resolve(true));
out.on('error', () => {
if (!stopDecoding) {
stopDecoding = true;
reject(new Error(`Failed to write CSV results to ${csvPath}`));
}
});
});
let nextOffset: number | undefined = 0;
do {
const chunk: DecodedBqrsChunk = await qs.cliServer.bqrsDecode(this.resultsPaths.resultsPath, resultSet, {
pageSize: 100,
offset: nextOffset,
});
chunk.tuples.forEach((tuple) => {
out.write(tuple.map((v, i) =>
chunk.columns[i].kind === 'String'
? `"${typeof v === 'string' ? v.replaceAll('"', '""') : v}"`
: v
).join(',') + '\n');
});
nextOffset = chunk.next;
} while (nextOffset && !stopDecoding);
out.end();
return promise;
}
/**
* Choose the name of the result set to run. If the `#select` set exists, use that. Otherwise,
* arbitrarily choose the first set. Most of the time, this will be correct.
*
* If the query has no result sets, then return undefined.
*/
async chooseResultSet(qs: qsClient.QueryServerClient) {
const resultSets = (await qs.cliServer.bqrsInfo(this.resultsPaths.resultsPath, 0))['result-sets'];
if (!resultSets.length) {
return undefined;
}
if (resultSets.find(r => r.name === SELECT_QUERY_NAME)) {
return SELECT_QUERY_NAME;
}
return resultSets[0].name;
}
/**
* Returns the path to the CSV alerts interpretation of this query results. If CSV results have
* not yet been produced, this will return first create the CSV results and then return the path.
*
* This method only works for queries with interpreted results.
*/
async ensureCsvAlerts(qs: qsClient.QueryServerClient, dbm: DatabaseManager): Promise<string> {
if (await this.hasCsv()) {
return this.csvPath;
}
const dbItem = dbm.findDatabaseItem(Uri.file(this.dbItemPath));
if (!dbItem) {
throw new Error(`Cannot produce CSV results because database is missing. ${this.dbItemPath}`);
}
let sourceInfo;
if (dbItem.sourceArchive !== undefined) {
sourceInfo = {
sourceArchive: dbItem.sourceArchive.fsPath,
sourceLocationPrefix: await dbItem.getSourceLocationPrefix(
qs.cliServer
),
};
}
await qs.cliServer.generateResultsCsv(ensureMetadataIsComplete(this.metadata), this.resultsPaths.resultsPath, this.csvPath, sourceInfo);
return this.csvPath;
}
/**
* Cleans this query's results directory.
*/
async deleteQuery(): Promise<void> {
await fs.remove(this.querySaveDir);
}
}
export interface QueryWithResults {
readonly query: QueryEvaluationInfo;
readonly result: messages.EvaluationResult;
readonly logFileLocation?: string;
readonly dispose: () => void;
}
export async function clearCacheInDatabase(
qs: qsClient.QueryServerClient,
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<messages.ClearCacheResult> {
if (dbItem.contents === undefined) {
throw new Error('Can\'t clear the cache in an invalid database.');
}
const db: messages.Dataset = {
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: 'default',
};
const params: messages.ClearCacheParams = {
dryRun: false,
db,
};
return qs.sendRequest(messages.clearCache, params, token, progress);
}
/**
* @param filePath This needs to be equivalent to Java's `Path.toRealPath(NO_FOLLOW_LINKS)`
*/
async function convertToQlPath(filePath: string): Promise<string> {
if (process.platform === 'win32') {
if (path.parse(filePath).root === filePath) {
// Java assumes uppercase drive letters are canonical.
return filePath.toUpperCase();
} else {
const dir = await convertToQlPath(path.dirname(filePath));
const fileName = path.basename(filePath);
const fileNames = await fs.readdir(dir);
for (const name of fileNames) {
// Leave the locale argument empty so that the default OS locale is used.
// We do this because this operation works on filesystem entities, which
// use the os locale, regardless of the locale of the running VS Code instance.
if (fileName.localeCompare(name, undefined, { sensitivity: 'accent' }) === 0) {
return path.join(dir, name);
}
}
}
throw new Error('Can\'t convert path to form suitable for QL:' + filePath);
} else {
return filePath;
}
}
/** Gets the selected position within the given editor. */
async function getSelectedPosition(editor: TextEditor, range?: Range): Promise<messages.Position> {
const selectedRange = range || editor.selection;
const pos = selectedRange.start;
const posEnd = selectedRange.end;
// Convert from 0-based to 1-based line and column numbers.
return {
fileName: await convertToQlPath(editor.document.fileName),
line: pos.line + 1,
column: pos.character + 1,
endLine: posEnd.line + 1,
endColumn: posEnd.character + 1
};
}
/**
* Compare the dbscheme implied by the query `query` and that of the current database.
* - If they are compatible, do nothing.
* - If they are incompatible but the database can be upgraded, suggest that upgrade.
* - If they are incompatible and the database cannot be upgraded, throw an error.
*/
async function checkDbschemeCompatibility(
cliServer: cli.CodeQLCliServer,
qs: qsClient.QueryServerClient,
query: QueryEvaluationInfo,
qlProgram: messages.QlProgram,
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<void> {
const searchPath = getOnDiskWorkspaceFolders();
if (dbItem.contents?.dbSchemeUri !== undefined) {
const { finalDbscheme } = await cliServer.resolveUpgrades(dbItem.contents.dbSchemeUri.fsPath, searchPath, false);
const hash = async function(filename: string): Promise<string> {
return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex');
};
// At this point, we have learned about three dbschemes:
// the dbscheme of the actual database we're querying.
const dbschemeOfDb = await hash(dbItem.contents.dbSchemeUri.fsPath);
// the dbscheme of the query we're running, including the library we've resolved it to use.
const dbschemeOfLib = await hash(query.queryDbscheme);
// the database we're able to upgrade to
const upgradableTo = await hash(finalDbscheme);
if (upgradableTo != dbschemeOfLib) {
reportNoUpgradePath(qlProgram, query);
}
if (upgradableTo == dbschemeOfLib &&
dbschemeOfDb != dbschemeOfLib) {
// Try to upgrade the database
await upgradeDatabaseExplicit(
qs,
dbItem,
progress,
token
);
}
}
}
function reportNoUpgradePath(qlProgram: messages.QlProgram, query: QueryEvaluationInfo): void {
throw new Error(
`Query ${qlProgram.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.`
);
}
/**
* Compile a non-destructive upgrade.
*/
async function compileNonDestructiveUpgrade(
qs: qsClient.QueryServerClient,
upgradeTemp: tmp.DirectoryResult,
query: QueryEvaluationInfo,
qlProgram: messages.QlProgram,
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<string> {
if (!dbItem?.contents?.dbSchemeUri) {
throw new Error('Database is invalid, and cannot be upgraded.');
}
// When packaging is used, dependencies may exist outside of the workspace and they are always on the resolved search path.
// When packaging is not used, all dependencies are in the workspace.
const upgradesPath = (await qs.cliServer.cliConstraints.supportsPackaging())
? qlProgram.libraryPath
: getOnDiskWorkspaceFolders();
const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(
dbItem.contents.dbSchemeUri.fsPath,
upgradesPath,
true,
query.queryDbscheme
);
if (!matchesTarget) {
reportNoUpgradePath(qlProgram, query);
}
const result = await compileDatabaseUpgradeSequence(qs, dbItem, scripts, upgradeTemp, progress, token);
if (result.compiledUpgrade === undefined) {
const error = result.error || '[no error message available]';
throw new Error(error);
}
// We can upgrade to the actual target
qlProgram.dbschemePath = query.queryDbscheme;
// We are new enough that we will always support single file upgrades.
return result.compiledUpgrade;
}
/**
* Prompts the user to save `document` if it has unsaved changes.
*
* @param document The document to save.
*
* @returns true if we should save changes and false if we should continue without saving changes.
* @throws UserCancellationException if we should abort whatever operation triggered this prompt
*/
async function promptUserToSaveChanges(document: TextDocument): Promise<boolean> {
if (document.isDirty) {
if (config.AUTOSAVE_SETTING.getValue()) {
return true;
}
else {
const yesItem = { title: 'Yes', isCloseAffordance: false };
const alwaysItem = { title: 'Always Save', isCloseAffordance: false };
const noItem = { title: 'No (run version on disk)', isCloseAffordance: false };
const cancelItem = { title: 'Cancel', isCloseAffordance: true };
const message = 'Query file has unsaved changes. Save now?';
const chosenItem = await window.showInformationMessage(
message,
{ modal: true },
yesItem, alwaysItem, noItem, cancelItem
);
if (chosenItem === alwaysItem) {
await config.AUTOSAVE_SETTING.updateValue(true, ConfigurationTarget.Workspace);
return true;
}
if (chosenItem === yesItem) {
return true;
}
if (chosenItem === cancelItem) {
throw new UserCancellationException('Query run cancelled.', true);
}
}
}
return false;
}
/**
* Determines which QL file to run during an invocation of `Run Query` or `Quick Evaluation`, as follows:
* - If the command was called by clicking on a file, then use that file.
* - Otherwise, use the file open in the current editor.
* - In either case, prompt the user to save the file if it is open with unsaved changes.
* - For `Quick Evaluation`, ensure the selected file is also the one open in the editor,
* and use the selected region.
* @param selectedResourceUri The selected resource when the command was run.
* @param quickEval Whether the command being run is `Quick Evaluation`.
*/
export async function determineSelectedQuery(
selectedResourceUri: Uri | undefined,
quickEval: boolean,
range?: Range
): Promise<SelectedQuery> {
const editor = window.activeTextEditor;
// Choose which QL file to use.
let queryUri: Uri;
if (selectedResourceUri) {
// A resource was passed to the command handler, so use it.
queryUri = selectedResourceUri;
} else {
// No resource was passed to the command handler, so obtain it from the active editor.
// This usually happens when the command is called from the Command Palette.
if (editor === undefined) {
throw new Error('No query was selected. Please select a query and try again.');
} else {
queryUri = editor.document.uri;
}
}
if (queryUri.scheme !== 'file') {
throw new Error('Can only run queries that are on disk.');
}
const queryPath = queryUri.fsPath;
if (quickEval) {
if (!(queryPath.endsWith('.ql') || queryPath.endsWith('.qll'))) {
throw new Error('The selected resource is not a CodeQL file; It should have the extension ".ql" or ".qll".');
}
} else {
if (!(queryPath.endsWith('.ql'))) {
throw new Error('The selected resource is not a CodeQL query file; It should have the extension ".ql".');
}
}
// Whether we chose the file from the active editor or from a context menu,
// if the same file is open with unsaved changes in the active editor,
// then prompt the user to save it first.
if (editor !== undefined && editor.document.uri.fsPath === queryPath) {
if (await promptUserToSaveChanges(editor.document)) {
await editor.document.save();
}
}
let quickEvalPosition: messages.Position | undefined = undefined;
let quickEvalText: string | undefined = undefined;
if (quickEval) {
if (editor == undefined) {
throw new Error('Can\'t run quick evaluation without an active editor.');
}
if (editor.document.fileName !== queryPath) {
// For Quick Evaluation we expect these to be the same.
// Report an error if we end up in this (hopefully unlikely) situation.
throw new Error('The selected resource for quick evaluation should match the active editor.');
}
quickEvalPosition = await getSelectedPosition(editor, range);
if (!editor.selection?.isEmpty) {
quickEvalText = editor.document.getText(editor.selection);
} else {
// capture the entire line if the user didn't select anything
const line = editor.document.lineAt(editor.selection.active.line);
quickEvalText = line.text.trim();
}
}
return { queryPath, quickEvalPosition, quickEvalText };
}
export async function compileAndRunQueryAgainstDatabase(
cliServer: cli.CodeQLCliServer,
qs: qsClient.QueryServerClient,
dbItem: DatabaseItem,
initialInfo: InitialQueryInfo,
queryStorageDir: string,
progress: ProgressCallback,
token: CancellationToken,
templates?: messages.TemplateDefinitions,
queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
): Promise<QueryWithResults> {
if (!dbItem.contents || !dbItem.contents.dbSchemeUri) {
throw new Error(`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`);
}
// Get the workspace folder paths.
const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
// Figure out the library path for the query.
const packConfig = await cliServer.resolveLibraryPath(diskWorkspaceFolders, initialInfo.queryPath);
if (!packConfig.dbscheme) {
throw new Error('Could not find a database scheme for this query. Please check that you have a valid qlpack.yml file for this query, which refers to a database scheme either in the `dbscheme` field or through one of its dependencies.');
}
// Check whether the query has an entirely different schema from the
// database. (Queries that merely need the database to be upgraded
// won't trigger this check)
// This test will produce confusing results if we ever change the name of the database schema files.
const querySchemaName = path.basename(packConfig.dbscheme);
const dbSchemaName = path.basename(dbItem.contents.dbSchemeUri.fsPath);
if (querySchemaName != dbSchemaName) {
void logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`);
throw new Error(`The query ${path.basename(initialInfo.queryPath)} cannot be run against the selected database (${dbItem.name}): their target languages are different. Please select a different database and try again.`);
}
const qlProgram: messages.QlProgram = {
// The project of the current document determines which library path
// we use. The `libraryPath` field in this server message is relative
// to the workspace root, not to the project root.
libraryPath: packConfig.libraryPath,
// Since we are compiling and running a query against a database,
// we use the database's DB scheme here instead of the DB scheme
// from the current document's project.
dbschemePath: dbItem.contents.dbSchemeUri.fsPath,
queryPath: initialInfo.queryPath
};
// Read the query metadata if possible, to use in the UI.
const metadata = await tryGetQueryMetadata(cliServer, qlProgram.queryPath);
let availableMlModels: cli.MlModelInfo[] = [];
if (!await cliServer.cliConstraints.supportsResolveMlModels()) {
void logger.log('Resolving ML models is unsupported by this version of the CLI. Running the query without any ML models.');
} else {
try {
availableMlModels = (await cliServer.resolveMlModels(diskWorkspaceFolders, initialInfo.queryPath)).models;
if (availableMlModels.length) {
void logger.log(`Found available ML models at the following paths: ${availableMlModels.map(x => `'${x.path}'`).join(', ')}.`);
} else {
void logger.log('Did not find any available ML models.');
}
} catch (e) {
const message = `Couldn't resolve available ML models for ${qlProgram.queryPath}. Running the ` +
`query without any ML models: ${e}.`;
void showAndLogErrorMessage(message);
}
}
const hasMetadataFile = (await dbItem.hasMetadataFile());
const query = new QueryEvaluationInfo(
path.join(queryStorageDir, initialInfo.id),
dbItem.databaseUri.fsPath,
hasMetadataFile,
packConfig.dbscheme,
initialInfo.quickEvalPosition,
metadata,
templates
);
await query.createTimestampFile();
let upgradeDir: tmp.DirectoryResult | undefined;
try {
let upgradeQlo;
if (await qs.cliServer.cliConstraints.supportsNonDestructiveUpgrades()) {
upgradeDir = await tmp.dir({ dir: upgradesTmpDir, unsafeCleanup: true });
upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, qlProgram, dbItem, progress, token);
} else {
await checkDbschemeCompatibility(cliServer, qs, query, qlProgram, dbItem, progress, token);
}
let errors;
try {
errors = await query.compile(qs, qlProgram, progress, token);
} catch (e) {
if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
return createSyntheticResult(query, 'Query cancelled', messages.QueryResultType.CANCELLATION);
} else {
throw e;
}
}
if (errors.length === 0) {
const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token, queryInfo);
if (result.resultType !== messages.QueryResultType.SUCCESS) {
const message = result.message || 'Failed to run query';
void logger.log(message);
void showAndLogErrorMessage(message);
}
return {
query,
result,
logFileLocation: result.logFileLocation,
dispose: () => {
qs.logger.removeAdditionalLogLocation(result.logFileLocation);
}
};
} else {
// Error dialogs are limited in size and scrollability,
// so we include a general description of the problem,
// and direct the user to the output window for the detailed compilation messages.
// However we don't show quick eval errors there so we need to display them anyway.
void qs.logger.log(
`Failed to compile query ${initialInfo.queryPath} against database scheme ${qlProgram.dbschemePath}:`,
{ additionalLogLocation: query.logPath }
);
const formattedMessages: string[] = [];
for (const error of errors) {
const message = error.message || '[no error message available]';
const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
formattedMessages.push(formatted);
void qs.logger.log(formatted, { additionalLogLocation: query.logPath });
}
if (initialInfo.isQuickEval && formattedMessages.length <= 2) {
// If there are more than 2 error messages, they will not be displayed well in a popup
// and will be trimmed by the function displaying the error popup. Accordingly, we only
// try to show the errors if there are 2 or less, otherwise we direct the user to the log.
void showAndLogErrorMessage('Quick evaluation compilation failed: ' + formattedMessages.join('\n'));
} else {
void showAndLogErrorMessage((initialInfo.isQuickEval ? 'Quick evaluation' : 'Query') + compilationFailedErrorTail);
}
return createSyntheticResult(query, 'Query had compilation errors', messages.QueryResultType.OTHER_ERROR);
}
} finally {
try {
await upgradeDir?.cleanup();
} catch (e) {
void qs.logger.log(
`Could not clean up the upgrades dir. Reason: ${getErrorMessage(e)}`,
{ additionalLogLocation: query.logPath }
);
}
}
}
/**
* Determines the initial information for a query. This is everything of interest
* we know about this query that is available before it is run.
*
* @param selectedQueryUri The Uri of the document containing the query to be run.
* @param databaseInfo The database to run the query against.
* @param isQuickEval true if this is a quick evaluation.
* @param range the selection range of the query to be run. Only used if isQuickEval is true.
* @returns The initial information for the query to be run.
*/
export async function createInitialQueryInfo(
selectedQueryUri: Uri | undefined,
databaseInfo: DatabaseInfo,
isQuickEval: boolean,
range?: Range
): Promise<InitialQueryInfo> {
// Determine which query to run, based on the selection and the active editor.
const { queryPath, quickEvalPosition, quickEvalText } = await determineSelectedQuery(selectedQueryUri, isQuickEval, range);
return {
queryPath,
isQuickEval,
isQuickQuery: isQuickQueryPath(queryPath),
databaseInfo,
id: `${path.basename(queryPath)}-${nanoid()}`,
start: new Date(),
... (isQuickEval ? {
queryText: quickEvalText!, // if this query is quick eval, it must have quick eval text
quickEvalPosition: quickEvalPosition
} : {
queryText: await fs.readFile(queryPath, 'utf8')
})
};
}
const compilationFailedErrorTail = ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
' and the query and database use the same target language. For more details on the error, go to View > Output,' +
' and choose CodeQL Query Server from the dropdown.';
/**
* Create a synthetic result for a query that failed to compile.
*/
function createSyntheticResult(
query: QueryEvaluationInfo,
message: string,
resultType: number
): QueryWithResults {
return {
query,
result: {
evaluationTime: 0,
resultType: resultType,
queryId: -1,
runId: -1,
message
},
dispose: () => { /**/ },
};
}

25
extensions/ql-vscode/src/stories/variant-analysis/VariantAnalysisLoading.stories.tsx Normal file
View File

@@ -0,0 +1,25 @@
import React from 'react';
import { ComponentMeta, ComponentStory } from '@storybook/react';
import { VariantAnalysisContainer } from '../../view/variant-analysis/VariantAnalysisContainer';
import { VariantAnalysisLoading as VariantAnalysisLoadingComponent } from '../../view/variant-analysis/VariantAnalysisLoading';
export default {
title: 'Variant Analysis/Variant Analysis Loading',
component: VariantAnalysisLoadingComponent,
decorators: [
(Story) => (
<VariantAnalysisContainer>
<Story />
</VariantAnalysisContainer>
)
],
argTypes: {}
} as ComponentMeta<typeof VariantAnalysisLoadingComponent>;
const Template: ComponentStory<typeof VariantAnalysisLoadingComponent> = () => (
<VariantAnalysisLoadingComponent />
);
export const VariantAnalysisLoading = Template.bind({});

17
extensions/ql-vscode/src/view/variant-analysis/VariantAnalysis.tsx
View File

@@ -9,6 +9,7 @@ import {
import { VariantAnalysisContainer } from './VariantAnalysisContainer';
import { VariantAnalysisHeader } from './VariantAnalysisHeader';
import { VariantAnalysisOutcomePanels } from './VariantAnalysisOutcomePanels';
import { VariantAnalysisLoading } from './VariantAnalysisLoading';
const variantAnalysis: VariantAnalysisDomainModel = {
id: 1,
@@ -157,9 +158,13 @@ const variantAnalysis: VariantAnalysisDomainModel = {
},
};
export function VariantAnalysis(): JSX.Element {
function getContainerContents(variantAnalysis: VariantAnalysisDomainModel) {
if (variantAnalysis.actionsWorkflowRunId === undefined) {
return <VariantAnalysisLoading />;
}
return (
<VariantAnalysisContainer>
<>
<VariantAnalysisHeader
variantAnalysis={variantAnalysis}
onOpenQueryFileClick={() => console.log('Open query')}
@@ -170,6 +175,14 @@ export function VariantAnalysis(): JSX.Element {
onViewLogsClick={() => console.log('View logs')}
/>
<VariantAnalysisOutcomePanels variantAnalysis={variantAnalysis} />
</>
);
}
export function VariantAnalysis(): JSX.Element {
return (
<VariantAnalysisContainer>
{getContainerContents(variantAnalysis)}
</VariantAnalysisContainer>
);
}

28
extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisLoading.tsx Normal file
View File

@@ -0,0 +1,28 @@
import * as React from 'react';
import styled from 'styled-components';
const Container = styled.div`
display: flex;
flex-direction: column;
align-items: center;
gap: 1em;
padding: 1em;
`;
const FirstRow = styled.div`
font-size: x-large;
font-weight: 600;
`;
const SecondRow = styled.div`
color: var(--vscode-descriptionForeground);
`;
export const VariantAnalysisLoading = () => {
return (
<Container>
<FirstRow>We are getting everything ready</FirstRow>
<SecondRow>Results will appear here shortly</SecondRow>
</Container>
);
};

15
extensions/ql-vscode/src/view/variant-analysis/__tests__/VariantAnalysisLoading.spec.tsx Normal file
View File

@@ -0,0 +1,15 @@
import * as React from 'react';
import { render as reactRender, screen } from '@testing-library/react';
import { VariantAnalysisLoading } from '../VariantAnalysisLoading';
describe(VariantAnalysisLoading.name, () => {
const render = () =>
reactRender(<VariantAnalysisLoading />);
it('renders loading text', async () => {
render();
expect(screen.getByText('We are getting everything ready')).toBeInTheDocument();
expect(screen.getByText('Results will appear here shortly')).toBeInTheDocument();
});
});

34
extensions/ql-vscode/src/vscode-tests/cli-integration/query.test.ts → extensions/ql-vscode/src/vscode-tests/cli-integration/legacy-query.test.ts
View File

@@ -4,14 +4,16 @@ import * as path from 'path';
import * as tmp from 'tmp';
import * as url from 'url';
import { CancellationTokenSource } from 'vscode-jsonrpc';
import * as messages from '../../pure/messages';
import * as qsClient from '../../queryserver-client';
import * as messages from '../../pure/legacy-messages';
import * as qsClient from '../../legacy-query-server/queryserver-client';
import * as cli from '../../cli';
import { CellValue } from '../../pure/bqrs-cli-types';
import { extensions } from 'vscode';
import { CodeQLExtensionInterface } from '../../extension';
import { fail } from 'assert';
import { skipIfNoCodeQL } from '../ensureCli';
import { QueryServerClient } from '../../legacy-query-server/queryserver-client';
import { logger, ProgressReporter } from '../../logging';
const baseDir = path.join(__dirname, '../../../test/data');
@@ -88,7 +90,7 @@ const db: messages.Dataset = {
workingSet: 'default',
};
describe('using the query server', function() {
describe('using the legacy query server', function() {
before(function() {
skipIfNoCodeQL(this);
});
@@ -97,28 +99,37 @@ describe('using the query server', function() {
// ensure they are all written with standard anonymous functions.
this.timeout(20000);
const nullProgressReporter: ProgressReporter = { report: () => { /** ignore */ } };
let qs: qsClient.QueryServerClient;
let cliServer: cli.CodeQLCliServer;
const queryServerStarted = new Checkpoint<void>();
beforeEach(async () => {
before(async () => {
try {
const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
if ('cliServer' in extension && 'qs' in extension) {
if ('cliServer' in extension) {
cliServer = extension.cliServer;
qs = extension.qs;
cliServer.quiet = true;
qs = new QueryServerClient({
codeQlPath: (await extension.distributionManager.getCodeQlPathWithoutVersionCheck()) || '',
debug: false,
cacheSize: 0,
numThreads: 1,
saveCache: false,
timeoutSecs: 0
}, cliServer, {
contextStoragePath: tmpDir.name,
logger
}, task => task(nullProgressReporter, new CancellationTokenSource().token));
await qs.startQueryServer();
} else {
throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
}
} catch (e) {
fail(e as Error);
}
});
it('should be able to start the query server', async function() {
await qs.startQueryServer();
await queryServerStarted.resolve();
});
for (const queryTestCase of queryTestCases) {
@@ -134,7 +145,6 @@ describe('using the query server', function() {
});
it(`should be able to compile query ${queryName}`, async function() {
await queryServerStarted.done();
expect(fs.existsSync(queryTestCase.queryPath)).to.be.true;
try {
const qlProgram: messages.QlProgram = {

21
extensions/ql-vscode/src/vscode-tests/cli-integration/queries.test.ts
View File

@@ -10,12 +10,11 @@ import { DatabaseItem, DatabaseManager } from '../../databases';
import { CodeQLExtensionInterface } from '../../extension';
import { cleanDatabases, dbLoc, storagePath } from './global.helper';
import { importArchiveDatabase } from '../../databaseFetcher';
import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo } from '../../run-queries';
import { CodeQLCliServer } from '../../cli';
import { QueryServerClient } from '../../queryserver-client';
import { skipIfNoCodeQL } from '../ensureCli';
import { QueryResultType } from '../../pure/messages';
import { tmpDir } from '../../helpers';
import { createInitialQueryInfo } from '../../run-queries-shared';
import { QueryRunner } from '../../queryRunner';
/**
@@ -31,7 +30,7 @@ describe('Queries', function() {
let dbItem: DatabaseItem;
let databaseManager: DatabaseManager;
let cli: CodeQLCliServer;
let qs: QueryServerClient;
let qs: QueryRunner;
let sandbox: sinon.SinonSandbox;
let progress: sinon.SinonSpy;
let token: CancellationToken;
@@ -104,9 +103,7 @@ describe('Queries', function() {
it('should run a query', async () => {
try {
const queryPath = path.join(__dirname, 'data', 'simple-query.ql');
const result = await compileAndRunQueryAgainstDatabase(
cli,
qs,
const result = qs.compileAndRunQueryAgainstDatabase(
dbItem,
await mockInitialQueryInfo(queryPath),
path.join(tmpDir.name, 'mock-storage-path'),
@@ -115,7 +112,7 @@ describe('Queries', function() {
);
// just check that the query was successful
expect(result.result.resultType).to.eq(QueryResultType.SUCCESS);
expect((await result).sucessful).to.eq(true);
} catch (e) {
console.error('Test Failed');
fail(e as Error);
@@ -127,9 +124,7 @@ describe('Queries', function() {
try {
await commands.executeCommand('codeQL.restartQueryServer');
const queryPath = path.join(__dirname, 'data', 'simple-query.ql');
const result = await compileAndRunQueryAgainstDatabase(
cli,
qs,
const result = await qs.compileAndRunQueryAgainstDatabase(
dbItem,
await mockInitialQueryInfo(queryPath),
path.join(tmpDir.name, 'mock-storage-path'),
@@ -137,9 +132,7 @@ describe('Queries', function() {
token
);
// this message would indicate that the databases were not properly reregistered
expect(result.result.message).not.to.eq('No result from server');
expect(result.result.resultType).to.eq(QueryResultType.SUCCESS);
expect(result.sucessful).to.eq(true);
} catch (e) {
console.error('Test Failed');
fail(e as Error);

428
extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/run-remote-query.test.ts
View File

@@ -11,8 +11,13 @@ import { Credentials } from '../../../authentication';
import { CliVersionConstraint, CodeQLCliServer } from '../../../cli';
import { CodeQLExtensionInterface } from '../../../extension';
import { setRemoteControllerRepo, setRemoteRepositoryLists } from '../../../config';
import * as config from '../../../config';
import { UserCancellationException } from '../../../commandRunner';
import * as ghApiClient from '../../../remote-queries/gh-api/gh-api-client';
import { lte } from 'semver';
import { VariantAnalysis } from '../../../remote-queries/gh-api/variant-analysis';
import { Repository } from '../../../remote-queries/gh-api/repository';
import { VariantAnalysisStatus } from '../../../remote-queries/shared/variant-analysis';
describe('Remote queries', function() {
const baseDir = path.join(__dirname, '../../../../src/vscode-tests/cli-integration');
@@ -27,6 +32,8 @@ describe('Remote queries', function() {
let token: CancellationToken;
let progress: sinon.SinonSpy;
let showQuickPickSpy: sinon.SinonStub;
let getRepositoryFromNwoStub: sinon.SinonStub;
let liveResultsStub: sinon.SinonStub;
// use `function` so we have access to `this`
beforeEach(async function() {
@@ -55,208 +62,309 @@ describe('Remote queries', function() {
.onFirstCall().resolves({ repositories: ['github/vscode-codeql'] } as unknown as QuickPickItem)
.onSecondCall().resolves('javascript' as unknown as QuickPickItem);
const dummyRepository: Repository = {
id: 123,
name: 'vscode-codeql',
full_name: 'github/vscode-codeql',
private: false,
};
getRepositoryFromNwoStub = sandbox.stub(ghApiClient, 'getRepositoryFromNwo').resolves(dummyRepository);
// always run in the vscode-codeql repo
await setRemoteControllerRepo('github/vscode-codeql');
await setRemoteRepositoryLists({ 'vscode-codeql': ['github/vscode-codeql'] });
liveResultsStub = sandbox.stub(config, 'isVariantAnalysisLiveResultsEnabled').returns(false);
});
afterEach(() => {
afterEach(async () => {
sandbox.restore();
});
it('should run a remote query that is part of a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack/in-pack.ql');
describe('when live results are not enabled', () => {
it('should run a remote query that is part of a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
printDirectoryContents(queryPackRootDir);
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
printDirectoryContents(queryPackRootDir);
// to retrieve the list of repositories
expect(showQuickPickSpy).to.have.been.calledOnce;
// to retrieve the list of repositories
expect(showQuickPickSpy).to.have.been.calledOnce;
// check a few files that we know should exist and others that we know should not
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
// the tarball to deliver to the server
expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
// check a few files that we know should exist and others that we know should not
const queryPackDir = path.join(queryPackRootDir, 'query-pack');
printDirectoryContents(queryPackDir);
// the tarball to deliver to the server
expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
expect(fs.existsSync(path.join(queryPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
const queryPackDir = path.join(queryPackRootDir, 'query-pack');
printDirectoryContents(queryPackDir);
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
expect(fs.existsSync(path.join(queryPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
// the compiled pack
const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
printDirectoryContents(compiledPackDir);
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
expect(fs.existsSync(path.join(compiledPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
// should have generated a correct qlpack file
const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
expect(qlpackContents.name).to.equal('codeql-remote/query');
// the compiled pack
const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
printDirectoryContents(compiledPackDir);
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'in-pack.ql', '0.0.0', await pathSerializationBroken());
expect(fs.existsSync(path.join(compiledPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
// should have generated a correct qlpack file
const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
expect(qlpackContents.name).to.equal('codeql-remote/query');
const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
const packNames = fs.readdirSync(libraryDir).sort();
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'in-pack.ql', '0.0.0', await pathSerializationBroken());
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
// later only have ['javascript-all']. ensure this test can handle either
expect(packNames.length).to.be.lessThan(3).and.greaterThan(0);
expect(packNames[0]).to.deep.equal('javascript-all');
const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
const packNames = fs.readdirSync(libraryDir).sort();
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
// later only have ['javascript-all']. ensure this test can handle either
expect(packNames.length).to.be.lessThan(3).and.greaterThan(0);
expect(packNames[0]).to.deep.equal('javascript-all');
});
it('should run a remote query that is not part of a qlpack', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
// to retrieve the list of repositories
// and a second time to ask for the language
expect(showQuickPickSpy).to.have.been.calledTwice;
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
// check a few files that we know should exist and others that we know should not
// the tarball to deliver to the server
printDirectoryContents(queryPackRootDir);
expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
const queryPackDir = path.join(queryPackRootDir, 'query-pack');
printDirectoryContents(queryPackDir);
expect(fs.existsSync(path.join(queryPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.false;
expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
// the compiled pack
const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
printDirectoryContents(compiledPackDir);
expect(fs.existsSync(path.join(compiledPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'in-pack.ql', '0.0.0', await pathSerializationBroken());
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.false;
expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
// should have generated a correct qlpack file
const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
expect(qlpackContents.name).to.equal('codeql-remote/query');
expect(qlpackContents.version).to.equal('0.0.0');
expect(qlpackContents.dependencies?.['codeql/javascript-all']).to.equal('*');
const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
printDirectoryContents(libraryDir);
const packNames = fs.readdirSync(libraryDir).sort();
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
// later only have ['javascript-all']. ensure this test can handle either
expect(packNames.length).to.be.lessThan(3).and.greaterThan(0);
expect(packNames[0]).to.deep.equal('javascript-all');
});
it('should run a remote query that is nested inside a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql');
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
// to retrieve the list of repositories
expect(showQuickPickSpy).to.have.been.calledOnce;
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
// check a few files that we know should exist and others that we know should not
// the tarball to deliver to the server
printDirectoryContents(queryPackRootDir);
expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
const queryPackDir = path.join(queryPackRootDir, 'query-pack');
printDirectoryContents(queryPackDir);
expect(fs.existsSync(path.join(queryPackDir, 'subfolder/in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'otherfolder/lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
// the compiled pack
const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
printDirectoryContents(compiledPackDir);
expect(fs.existsSync(path.join(compiledPackDir, 'otherfolder/lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'subfolder/in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'subfolder/in-pack.ql', '0.0.0', await pathSerializationBroken());
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
// should have generated a correct qlpack file
const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
expect(qlpackContents.name).to.equal('codeql-remote/query');
expect(qlpackContents.version).to.equal('0.0.0');
expect(qlpackContents.dependencies?.['codeql/javascript-all']).to.equal('*');
const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
printDirectoryContents(libraryDir);
const packNames = fs.readdirSync(libraryDir).sort();
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
// later only have ['javascript-all']. ensure this test can handle either
expect(packNames.length).to.be.lessThan(3).and.greaterThan(0);
expect(packNames[0]).to.deep.equal('javascript-all');
});
it('should cancel a run before uploading', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, token);
token.isCancellationRequested = true;
try {
await promise;
assert.fail('should have thrown');
} catch (e) {
expect(e).to.be.instanceof(UserCancellationException);
}
});
});
it('should run a remote query that is not part of a qlpack', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
describe('when live results are enabled', () => {
beforeEach(() => {
liveResultsStub.returns(true);
});
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
const dummyVariantAnalysis: VariantAnalysis = {
id: 123,
controller_repo: {
id: 64,
name: 'pickles',
full_name: 'github/pickles',
private: false,
},
actor_id: 27,
query_language: 'javascript',
query_pack_url: 'https://example.com/foo',
status: 'in_progress',
};
// to retrieve the list of repositories
// and a second time to ask for the language
expect(showQuickPickSpy).to.have.been.calledTwice;
it('should run a variant analysis that is part of a qlpack', async () => {
const submitVariantAnalysisStub = sandbox.stub(ghApiClient, 'submitVariantAnalysis').resolves(dummyVariantAnalysis);
// check a few files that we know should exist and others that we know should not
const fileUri = getFile('data-remote-qlpack/in-pack.ql');
// the tarball to deliver to the server
printDirectoryContents(queryPackRootDir);
expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
expect(querySubmissionResult).to.be.ok;
const variantAnalysis = querySubmissionResult!.variantAnalysis!;
expect(variantAnalysis.id).to.be.equal(dummyVariantAnalysis.id);
expect(variantAnalysis.status).to.be.equal(VariantAnalysisStatus.InProgress);
const queryPackDir = path.join(queryPackRootDir, 'query-pack');
printDirectoryContents(queryPackDir);
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
expect(fs.existsSync(path.join(queryPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.false;
expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
expect(submitVariantAnalysisStub).to.have.been.calledOnce;
});
// the compiled pack
const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
printDirectoryContents(compiledPackDir);
expect(fs.existsSync(path.join(compiledPackDir, 'in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'in-pack.ql', '0.0.0', await pathSerializationBroken());
it('should run a remote query that is not part of a qlpack', async () => {
const submitVariantAnalysisStub = sandbox.stub(ghApiClient, 'submitVariantAnalysis').resolves(dummyVariantAnalysis);
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.false;
expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
// should have generated a correct qlpack file
const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
expect(qlpackContents.name).to.equal('codeql-remote/query');
expect(qlpackContents.version).to.equal('0.0.0');
expect(qlpackContents.dependencies?.['codeql/javascript-all']).to.equal('*');
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
printDirectoryContents(libraryDir);
const packNames = fs.readdirSync(libraryDir).sort();
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
expect(querySubmissionResult).to.be.ok;
const variantAnalysis = querySubmissionResult!.variantAnalysis!;
expect(variantAnalysis.id).to.be.equal(dummyVariantAnalysis.id);
expect(variantAnalysis.status).to.be.equal(VariantAnalysisStatus.InProgress);
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
// later only have ['javascript-all']. ensure this test can handle either
expect(packNames.length).to.be.lessThan(3).and.greaterThan(0);
expect(packNames[0]).to.deep.equal('javascript-all');
});
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
it('should run a remote query that is nested inside a qlpack', async () => {
const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql');
expect(submitVariantAnalysisStub).to.have.been.calledOnce;
});
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
expect(querySubmissionResult).to.be.ok;
const queryPackRootDir = querySubmissionResult!.queryDirPath!;
it('should run a remote query that is nested inside a qlpack', async () => {
const submitVariantAnalysisStub = sandbox.stub(ghApiClient, 'submitVariantAnalysis').resolves(dummyVariantAnalysis);
// to retrieve the list of repositories
expect(showQuickPickSpy).to.have.been.calledOnce;
const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql');
// check a few files that we know should exist and others that we know should not
const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
expect(querySubmissionResult).to.be.ok;
const variantAnalysis = querySubmissionResult!.variantAnalysis!;
expect(variantAnalysis.id).to.be.equal(dummyVariantAnalysis.id);
expect(variantAnalysis.status).to.be.equal(VariantAnalysisStatus.InProgress);
// the tarball to deliver to the server
printDirectoryContents(queryPackRootDir);
expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
const queryPackDir = path.join(queryPackRootDir, 'query-pack');
printDirectoryContents(queryPackDir);
expect(submitVariantAnalysisStub).to.have.been.calledOnce;
});
expect(fs.existsSync(path.join(queryPackDir, 'subfolder/in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'otherfolder/lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
it('should cancel a run before uploading', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
// the compiled pack
const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
printDirectoryContents(compiledPackDir);
expect(fs.existsSync(path.join(compiledPackDir, 'otherfolder/lib.qll'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'subfolder/in-pack.ql'))).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'subfolder/in-pack.ql', '0.0.0', await pathSerializationBroken());
const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, token);
// depending on the cli version, we should have one of these files
expect(
fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
).to.be.true;
expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
// should have generated a correct qlpack file
const qlpackContents: any = yaml.load(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
expect(qlpackContents.name).to.equal('codeql-remote/query');
expect(qlpackContents.version).to.equal('0.0.0');
expect(qlpackContents.dependencies?.['codeql/javascript-all']).to.equal('*');
token.isCancellationRequested = true;
const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
printDirectoryContents(libraryDir);
const packNames = fs.readdirSync(libraryDir).sort();
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
// later only have ['javascript-all']. ensure this test can handle either
expect(packNames.length).to.be.lessThan(3).and.greaterThan(0);
expect(packNames[0]).to.deep.equal('javascript-all');
});
it('should cancel a run before uploading', async () => {
const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, token);
token.isCancellationRequested = true;
try {
await promise;
assert.fail('should have thrown');
} catch (e) {
expect(e).to.be.instanceof(UserCancellationException);
}
try {
await promise;
assert.fail('should have thrown');
} catch (e) {
expect(e).to.be.instanceof(UserCancellationException);
}
});
});
function verifyQlPack(qlpackPath: string, queryPath: string, packVersion: string, pathSerializationBroken: boolean) {

50
extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts
View File

@@ -14,12 +14,11 @@ import {
findSourceArchive
} from '../../databases';
import { Logger } from '../../logging';
import { QueryServerClient } from '../../queryserver-client';
import { registerDatabases } from '../../pure/messages';
import { ProgressCallback } from '../../commandRunner';
import { CodeQLCliServer } from '../../cli';
import { encodeArchiveBasePath, encodeSourceArchiveUri } from '../../archive-filesystem-provider';
import { testDisposeHandler } from '../test-dispose-handler';
import { QueryRunner } from '../../queryRunner';
describe('databases', () => {
@@ -33,7 +32,8 @@ describe('databases', () => {
let updateSpy: sinon.SinonSpy;
let getSpy: sinon.SinonStub;
let dbChangedHandler: sinon.SinonSpy;
let sendRequestSpy: sinon.SinonSpy;
let registerSpy: sinon.SinonSpy;
let deregisterSpy: sinon.SinonSpy;
let supportsDatabaseRegistrationSpy: sinon.SinonStub;
let supportsLanguageNameSpy: sinon.SinonStub;
let resolveDatabaseSpy: sinon.SinonStub;
@@ -48,7 +48,8 @@ describe('databases', () => {
updateSpy = sandbox.spy();
getSpy = sandbox.stub();
getSpy.returns([]);
sendRequestSpy = sandbox.stub();
registerSpy = sandbox.stub();
deregisterSpy = sandbox.stub();
dbChangedHandler = sandbox.spy();
supportsDatabaseRegistrationSpy = sandbox.stub();
supportsDatabaseRegistrationSpy.resolves(true);
@@ -65,9 +66,10 @@ describe('databases', () => {
storagePath: dir.name
} as unknown as ExtensionContext,
{
sendRequest: sendRequestSpy,
onDidStartQueryServer: () => { /**/ }
} as unknown as QueryServerClient,
registerDatabase: registerSpy,
deregisterDatabase: deregisterSpy,
onStart: () => { /**/ }
} as unknown as QueryRunner,
{
cliConstraints: {
supportsLanguageName: supportsLanguageNameSpy,
@@ -259,12 +261,6 @@ describe('databases', () => {
// similar test as above, but also check the call to sendRequestSpy to make sure they send the
// registration messages.
const mockDbItem = createMockDB();
const registration = {
databases: [{
dbDir: mockDbItem.contents!.datasetUri.fsPath,
workingSet: 'default'
}]
};
sandbox.stub(fs, 'remove').resolves();
@@ -274,7 +270,7 @@ describe('databases', () => {
mockDbItem
);
// Should have registered this database
expect(sendRequestSpy).to.have.been.calledWith(registerDatabases, registration, {}, {});
expect(registerSpy).to.have.been.calledWith({}, {}, mockDbItem);
await databaseManager.removeDatabaseItem(
{} as ProgressCallback,
@@ -283,31 +279,7 @@ describe('databases', () => {
);
// Should have deregistered this database
expect(sendRequestSpy).to.have.been.calledWith(registerDatabases, registration, {}, {});
});
it('should avoid registration when query server does not support it', async () => {
// similar test as above, but now pretend query server doesn't support registration
supportsDatabaseRegistrationSpy.resolves(false);
const mockDbItem = createMockDB();
sandbox.stub(fs, 'remove').resolves();
await (databaseManager as any).addDatabaseItem(
{} as ProgressCallback,
{} as CancellationToken,
mockDbItem
);
// Should NOT have registered this database
expect(sendRequestSpy).not.to.have.been.called;
await databaseManager.removeDatabaseItem(
{} as ProgressCallback,
{} as CancellationToken,
mockDbItem
);
// Should NOT have deregistered this database
expect(sendRequestSpy).not.to.have.been.called;
expect(deregisterSpy).to.have.been.calledWith({}, {}, mockDbItem);
});
});

2
extensions/ql-vscode/src/vscode-tests/minimal-workspace/determining-selected-query-test.ts
View File

@@ -2,7 +2,7 @@ import { expect } from 'chai';
import * as path from 'path';
import * as vscode from 'vscode';
import { Uri } from 'vscode';
import { determineSelectedQuery } from '../../run-queries';
import { determineSelectedQuery } from '../../run-queries-shared';
async function showQlDocument(name: string): Promise<vscode.TextDocument> {
const folderPath = vscode.workspace.workspaceFolders![0].uri.fsPath;

2
extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/astBuilder.test.ts
View File

@@ -3,10 +3,10 @@ import { expect } from 'chai';
import * as sinon from 'sinon';
import AstBuilder from '../../../contextual/astBuilder';
import { QueryWithResults } from '../../../run-queries';
import { CodeQLCliServer } from '../../../cli';
import { DatabaseItem } from '../../../databases';
import { Uri } from 'vscode';
import { QueryWithResults } from '../../../run-queries-shared';
/**
*

14
extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts
View File

@@ -7,10 +7,8 @@ import * as sinon from 'sinon';
import { logger } from '../../logging';
import { registerQueryHistoryScubber } from '../../query-history-scrubber';
import { QueryHistoryManager, HistoryTreeDataProvider, SortOrder } from '../../query-history';
import { QueryEvaluationInfo, QueryWithResults } from '../../run-queries';
import { QueryEvaluationInfo, QueryWithResults } from '../../run-queries-shared';
import { QueryHistoryConfig, QueryHistoryConfigListener } from '../../config';
import * as messages from '../../pure/messages';
import { QueryServerClient } from '../../queryserver-client';
import { LocalQueryInfo, InitialQueryInfo } from '../../query-results';
import { DatabaseManager } from '../../databases';
import * as tmp from 'tmp-promise';
@@ -21,6 +19,7 @@ import { HistoryItemLabelProvider } from '../../history-item-label-provider';
import { RemoteQueriesManager } from '../../remote-queries/remote-queries-manager';
import { ResultsView } from '../../interface';
import { EvalLogViewer } from '../../eval-log-viewer';
import { QueryRunner } from '../../queryRunner';
describe('query-history', () => {
const mockExtensionLocation = path.join(tmpDir.name, 'mock-extension-location');
@@ -785,18 +784,15 @@ describe('query-history', () => {
hasInterpretedResults: () => Promise.resolve(hasInterpretedResults),
deleteQuery: sandbox.stub(),
} as unknown as QueryEvaluationInfo,
result: {
resultType: didRunSuccessfully
? messages.QueryResultType.SUCCESS
: messages.QueryResultType.OTHER_ERROR
} as messages.EvaluationResult,
sucessful: didRunSuccessfully,
message: 'foo',
dispose: sandbox.spy()
};
}
async function createMockQueryHistory(allHistory: LocalQueryInfo[]) {
const qhm = new QueryHistoryManager(
{} as QueryServerClient,
{} as QueryRunner,
{} as DatabaseManager,
localQueriesResultsViewStub,
remoteQueriesManagerStub,

60
extensions/ql-vscode/src/vscode-tests/no-workspace/query-results.test.ts
View File

@@ -3,13 +3,14 @@ import * as path from 'path';
import * as fs from 'fs-extra';
import * as sinon from 'sinon';
import { LocalQueryInfo, InitialQueryInfo, interpretResultsSarif } from '../../query-results';
import { QueryEvaluationInfo, QueryWithResults } from '../../run-queries';
import { EvaluationResult, QueryResultType } from '../../pure/messages';
import { QueryWithResults } from '../../run-queries-shared';
import { DatabaseInfo, SortDirection, SortedResultSetInfo } from '../../pure/interface-types';
import { CodeQLCliServer, SourceInfo } from '../../cli';
import { CancellationTokenSource, Uri } from 'vscode';
import { tmpDir } from '../../helpers';
import { slurpQueryHistory, splatQueryHistory } from '../../query-serialization';
import { formatLegacyMessage, QueryInProgress } from '../../legacy-query-server/run-queries';
import { EvaluationResult, QueryResultType } from '../../pure/legacy-messages';
describe('query-results', () => {
let disposeSpy: sinon.SinonSpy;
@@ -87,30 +88,34 @@ describe('query-results', () => {
expect(completedQuery.getResultsPath('zxa')).to.eq('bxa');
});
it('should get the statusString', () => {
const fqi = createMockFullQueryInfo('a', createMockQueryWithResults(queryPath, false));
const completedQuery = fqi.completedQuery!;
it('should format the statusString', () => {
completedQuery.result.message = 'Tremendously';
expect(completedQuery.statusString).to.eq('failed: Tremendously');
const evalResult: EvaluationResult = {
resultType: QueryResultType.OTHER_ERROR,
evaluationTime: 12340,
queryId: 3,
runId: 1,
};
completedQuery.result.resultType = QueryResultType.OTHER_ERROR;
expect(completedQuery.statusString).to.eq('failed: Tremendously');
evalResult.message = 'Tremendously';
expect(formatLegacyMessage(evalResult)).to.eq('failed: Tremendously');
completedQuery.result.resultType = QueryResultType.CANCELLATION;
completedQuery.result.evaluationTime = 2345;
expect(completedQuery.statusString).to.eq('cancelled after 2 seconds');
evalResult.resultType = QueryResultType.OTHER_ERROR;
expect(formatLegacyMessage(evalResult)).to.eq('failed: Tremendously');
completedQuery.result.resultType = QueryResultType.OOM;
expect(completedQuery.statusString).to.eq('out of memory');
evalResult.resultType = QueryResultType.CANCELLATION;
evalResult.evaluationTime = 2345;
expect(formatLegacyMessage(evalResult)).to.eq('cancelled after 2 seconds');
completedQuery.result.resultType = QueryResultType.SUCCESS;
expect(completedQuery.statusString).to.eq('finished in 2 seconds');
evalResult.resultType = QueryResultType.OOM;
expect(formatLegacyMessage(evalResult)).to.eq('out of memory');
completedQuery.result.resultType = QueryResultType.TIMEOUT;
expect(completedQuery.statusString).to.eq('timed out after 2 seconds');
evalResult.resultType = QueryResultType.SUCCESS;
expect(formatLegacyMessage(evalResult)).to.eq('finished in 2 seconds');
evalResult.resultType = QueryResultType.TIMEOUT;
expect(formatLegacyMessage(evalResult)).to.eq('timed out after 2 seconds');
});
it('should updateSortState', async () => {
// setup
const fqi = createMockFullQueryInfo('a', createMockQueryWithResults(queryPath));
@@ -267,9 +272,6 @@ describe('query-results', () => {
if (!('quickEvalPosition' in query)) {
(query as any).quickEvalPosition = undefined;
}
if (!('templates' in query)) {
(query as any).templates = undefined;
}
}
});
expectedHistory.forEach(info => {
@@ -310,7 +312,7 @@ describe('query-results', () => {
fs.mkdirpSync(queryPath);
fs.writeFileSync(resultsPath, '', 'utf8');
const query = new QueryEvaluationInfo(
const query = new QueryInProgress(
queryPath,
Uri.file(dbPath).fsPath,
true,
@@ -321,14 +323,10 @@ describe('query-results', () => {
},
);
const result = {
query,
result: {
evaluationTime: 12340,
resultType: didRunSuccessfully
? QueryResultType.SUCCESS
: QueryResultType.OTHER_ERROR
} as EvaluationResult,
const result: QueryWithResults = {
query: query.queryEvalInfo,
sucessful: didRunSuccessfully,
message: 'foo',
dispose: disposeSpy,
};

4
extensions/ql-vscode/src/vscode-tests/no-workspace/remote-queries/remote-query-history.test.ts
View File

@@ -8,7 +8,6 @@ import { QueryHistoryConfig } from '../../../config';
import { DatabaseManager } from '../../../databases';
import { tmpDir } from '../../../helpers';
import { QueryHistoryManager } from '../../../query-history';
import { QueryServerClient } from '../../../queryserver-client';
import { Credentials } from '../../../authentication';
import { AnalysesResultsManager } from '../../../remote-queries/analyses-results-manager';
import { RemoteQueryResult } from '../../../remote-queries/shared/remote-query-result';
@@ -20,6 +19,7 @@ import { HistoryItemLabelProvider } from '../../../history-item-label-provider';
import { RemoteQueriesManager } from '../../../remote-queries/remote-queries-manager';
import { ResultsView } from '../../../interface';
import { EvalLogViewer } from '../../../eval-log-viewer';
import { QueryRunner } from '../../../queryRunner';
/**
* Tests for remote queries and how they interact with the query history manager.
@@ -90,7 +90,7 @@ describe('Remote queries and query history manager', function() {
remoteQueryResult1 = fs.readJSONSync(path.join(STORAGE_DIR, 'queries', rawQueryHistory[1].queryId, 'query-result.json'));
qhm = new QueryHistoryManager(
{} as QueryServerClient,
{} as QueryRunner,
{} as DatabaseManager,
localQueriesResultsViewStub,
remoteQueriesManagerStub,

23
extensions/ql-vscode/src/vscode-tests/no-workspace/remote-queries/run-remote-query.test.ts
View File

@@ -1,11 +1,18 @@
import { expect } from 'chai';
import * as os from 'os';
import { parseResponse } from '../../../remote-queries/run-remote-query';
import { Repository } from '../../../remote-queries/shared/repository';
describe('run-remote-query', () => {
describe('parseResponse', () => {
const controllerRepository: Repository = {
id: 123,
fullName: 'org/name',
private: true
};
it('should parse a successful response', () => {
const result = parseResponse('org', 'name', {
const result = parseResponse(controllerRepository, {
workflow_run_id: 123,
repositories_queried: ['a/b', 'c/d'],
});
@@ -20,7 +27,7 @@ describe('run-remote-query', () => {
});
it('should parse a response with invalid repos', () => {
const result = parseResponse('org', 'name', {
const result = parseResponse(controllerRepository, {
workflow_run_id: 123,
repositories_queried: ['a/b', 'c/d'],
errors: {
@@ -47,7 +54,7 @@ describe('run-remote-query', () => {
});
it('should parse a response with repos w/o databases', () => {
const result = parseResponse('org', 'name', {
const result = parseResponse(controllerRepository, {
workflow_run_id: 123,
repositories_queried: ['a/b', 'c/d'],
errors: {
@@ -75,7 +82,7 @@ describe('run-remote-query', () => {
});
it('should parse a response with private repos', () => {
const result = parseResponse('org', 'name', {
const result = parseResponse(controllerRepository, {
workflow_run_id: 123,
repositories_queried: ['a/b', 'c/d'],
errors: {
@@ -103,7 +110,7 @@ describe('run-remote-query', () => {
});
it('should parse a response with cutoff repos and cutoff repos count', () => {
const result = parseResponse('org', 'name', {
const result = parseResponse(controllerRepository, {
workflow_run_id: 123,
repositories_queried: ['a/b', 'c/d'],
errors: {
@@ -132,7 +139,7 @@ describe('run-remote-query', () => {
});
it('should parse a response with cutoff repos count but not cutoff repos', () => {
const result = parseResponse('org', 'name', {
const result = parseResponse(controllerRepository, {
workflow_run_id: 123,
repositories_queried: ['a/b', 'c/d'],
errors: {
@@ -159,7 +166,7 @@ describe('run-remote-query', () => {
});
it('should parse a response with invalid repos and repos w/o databases', () => {
const result = parseResponse('org', 'name', {
const result = parseResponse(controllerRepository, {
workflow_run_id: 123,
repositories_queried: ['a/b', 'c/d'],
errors: {
@@ -191,7 +198,7 @@ describe('run-remote-query', () => {
});
it('should parse a response with one repo of each category, and not pluralize "repositories"', () => {
const result = parseResponse('org', 'name', {
const result = parseResponse(controllerRepository, {
workflow_run_id: 123,
repositories_queried: ['a/b'],
errors: {

201
extensions/ql-vscode/src/vscode-tests/no-workspace/run-queries.test.ts
View File

@@ -4,13 +4,15 @@ import * as fs from 'fs-extra';
import * as sinon from 'sinon';
import { Uri } from 'vscode';
import { QueryEvaluationInfo } from '../../run-queries';
import { Severity, compileQuery } from '../../pure/messages';
import { Severity, compileQuery, registerDatabases, deregisterDatabases } from '../../pure/legacy-messages';
import * as config from '../../config';
import { tmpDir } from '../../helpers';
import { QueryServerClient } from '../../queryserver-client';
import { QueryServerClient } from '../../legacy-query-server/queryserver-client';
import { CodeQLCliServer } from '../../cli';
import { SELECT_QUERY_NAME } from '../../contextual/locationFinder';
import { QueryInProgress } from '../../legacy-query-server/run-queries';
import { LegacyQueryRunner } from '../../legacy-query-server/legacyRunner';
import { DatabaseItem } from '../../databases';
describe('run-queries', () => {
let sandbox: sinon.SinonSandbox;
@@ -29,55 +31,53 @@ describe('run-queries', () => {
const info = createMockQueryInfo(true, saveDir);
expect(info.compiledQueryPath).to.eq(path.join(saveDir, 'compiledQuery.qlo'));
expect(info.dilPath).to.eq(path.join(saveDir, 'results.dil'));
expect(info.resultsPaths.resultsPath).to.eq(path.join(saveDir, 'results.bqrs'));
expect(info.resultsPaths.interpretedResultsPath).to.eq(path.join(saveDir, 'interpretedResults.sarif'));
expect(info.queryEvalInfo.dilPath).to.eq(path.join(saveDir, 'results.dil'));
expect(info.queryEvalInfo.resultsPaths.resultsPath).to.eq(path.join(saveDir, 'results.bqrs'));
expect(info.queryEvalInfo.resultsPaths.interpretedResultsPath).to.eq(path.join(saveDir, 'interpretedResults.sarif'));
expect(info.dbItemPath).to.eq(Uri.file('/abc').fsPath);
});
it('should check if interpreted results can be created', async () => {
const info = createMockQueryInfo(true);
expect(info.canHaveInterpretedResults()).to.eq(true);
expect(info.queryEvalInfo.canHaveInterpretedResults(), '1').to.eq(true);
(info as any).databaseHasMetadataFile = false;
expect(info.canHaveInterpretedResults()).to.eq(false);
(info.queryEvalInfo as any).databaseHasMetadataFile = false;
expect(info.queryEvalInfo.canHaveInterpretedResults(), '2').to.eq(false);
(info as any).databaseHasMetadataFile = true;
(info.queryEvalInfo as any).databaseHasMetadataFile = true;
info.metadata!.kind = undefined;
expect(info.canHaveInterpretedResults()).to.eq(false);
expect(info.queryEvalInfo.canHaveInterpretedResults(), '3').to.eq(false);
info.metadata!.kind = 'table';
expect(info.canHaveInterpretedResults()).to.eq(false);
expect(info.queryEvalInfo.canHaveInterpretedResults(), '4').to.eq(false);
// Graphs are not interpreted unless canary is set
info.metadata!.kind = 'graph';
expect(info.canHaveInterpretedResults()).to.eq(false);
expect(info.queryEvalInfo.canHaveInterpretedResults(), '5').to.eq(false);
(config.isCanary as sinon.SinonStub).returns(true);
expect(info.canHaveInterpretedResults()).to.eq(true);
expect(info.queryEvalInfo.canHaveInterpretedResults(), '6').to.eq(true);
});
[SELECT_QUERY_NAME, 'other'].forEach(resultSetName => {
it(`should export csv results for result set ${resultSetName}`, async () => {
const csvLocation = path.join(tmpDir.name, 'test.csv');
const qs = createMockQueryServerClient(
createMockCliServer({
bqrsInfo: [{ 'result-sets': [{ name: resultSetName }, { name: 'hucairz' }] }],
bqrsDecode: [{
columns: [{ kind: 'NotString' }, { kind: 'String' }],
tuples: [['a', 'b'], ['c', 'd']],
next: 1
}, {
// just for fun, give a different set of columns here
// this won't happen with the real CLI, but it's a good test
columns: [{ kind: 'String' }, { kind: 'NotString' }, { kind: 'StillNotString' }],
tuples: [['a', 'b', 'c']]
}]
})
);
const cliServer = createMockCliServer({
bqrsInfo: [{ 'result-sets': [{ name: resultSetName }, { name: 'hucairz' }] }],
bqrsDecode: [{
columns: [{ kind: 'NotString' }, { kind: 'String' }],
tuples: [['a', 'b'], ['c', 'd']],
next: 1
}, {
// just for fun, give a different set of columns here
// this won't happen with the real CLI, but it's a good test
columns: [{ kind: 'String' }, { kind: 'NotString' }, { kind: 'StillNotString' }],
tuples: [['a', 'b', 'c']]
}]
});
const info = createMockQueryInfo();
const promise = info.exportCsvResults(qs, csvLocation);
const promise = info.queryEvalInfo.exportCsvResults(cliServer, csvLocation);
const result = await promise;
expect(result).to.eq(true);
@@ -86,14 +86,14 @@ describe('run-queries', () => {
expect(csv).to.eq('a,"b"\nc,"d"\n"a",b,c\n');
// now verify that we are using the expected result set
expect((qs.cliServer.bqrsDecode as sinon.SinonStub).callCount).to.eq(2);
expect((qs.cliServer.bqrsDecode as sinon.SinonStub).getCall(0).args[1]).to.eq(resultSetName);
expect((cliServer.bqrsDecode as sinon.SinonStub).callCount).to.eq(2);
expect((cliServer.bqrsDecode as sinon.SinonStub).getCall(0).args[1]).to.eq(resultSetName);
});
});
it('should export csv results with characters that need to be escaped', async () => {
const csvLocation = path.join(tmpDir.name, 'test.csv');
const qs = createMockQueryServerClient(
const cliServer =
createMockCliServer({
bqrsInfo: [{ 'result-sets': [{ name: SELECT_QUERY_NAME }, { name: 'hucairz' }] }],
bqrsDecode: [{
@@ -109,10 +109,9 @@ describe('run-queries', () => {
[123.98, 456.99],
],
}]
})
);
});
const info = createMockQueryInfo();
const promise = info.exportCsvResults(qs, csvLocation);
const promise = info.queryEvalInfo.exportCsvResults(cliServer, csvLocation);
const result = await promise;
expect(result).to.eq(true);
@@ -121,19 +120,18 @@ describe('run-queries', () => {
expect(csv).to.eq('"a","""b"""\nc,xxx,"d,yyy"\naaa " bbb,"ccc "" ddd"\ntrue,"false"\n123,"456"\n123.98,"456.99"\n');
// now verify that we are using the expected result set
expect((qs.cliServer.bqrsDecode as sinon.SinonStub).callCount).to.eq(1);
expect((qs.cliServer.bqrsDecode as sinon.SinonStub).getCall(0).args[1]).to.eq(SELECT_QUERY_NAME);
expect((cliServer.bqrsDecode as sinon.SinonStub).callCount).to.eq(1);
expect((cliServer.bqrsDecode as sinon.SinonStub).getCall(0).args[1]).to.eq(SELECT_QUERY_NAME);
});
it('should handle csv exports for a query with no result sets', async () => {
const csvLocation = path.join(tmpDir.name, 'test.csv');
const qs = createMockQueryServerClient(
const cliServer =
createMockCliServer({
bqrsInfo: [{ 'result-sets': [] }]
})
);
});
const info = createMockQueryInfo();
const result = await info.exportCsvResults(qs, csvLocation);
const result = await info.queryEvalInfo.exportCsvResults(cliServer, csvLocation);
expect(result).to.eq(false);
});
@@ -187,9 +185,126 @@ describe('run-queries', () => {
});
});
describe('register', () => {
it('should register', async () => {
const qs = createMockQueryServerClient(
{
cliConstraints: {
supportsDatabaseRegistration: () => true
}
} as any);
const runner = new LegacyQueryRunner(qs);
const mockProgress = 'progress-monitor';
const mockCancel = 'cancel-token';
const datasetUri = Uri.file('dataset-uri');
const dbItem: DatabaseItem = {
contents: {
datasetUri
}
} as any;
await runner.registerDatabase(mockProgress as any, mockCancel as any, dbItem);
expect(qs.sendRequest).to.have.been.calledOnceWith(
registerDatabases,
{
databases: [
{
dbDir: datasetUri.fsPath,
workingSet: 'default'
}
]
},
mockCancel,
mockProgress
);
});
it('should deregister', async () => {
const qs = createMockQueryServerClient(
{
cliConstraints: {
supportsDatabaseRegistration: () => true
}
} as any);
const runner = new LegacyQueryRunner(qs);
const mockProgress = 'progress-monitor';
const mockCancel = 'cancel-token';
const datasetUri = Uri.file('dataset-uri');
const dbItem: DatabaseItem = {
contents: {
datasetUri
}
} as any;
await runner.deregisterDatabase(mockProgress as any, mockCancel as any, dbItem);
expect(qs.sendRequest).to.have.been.calledOnceWith(
deregisterDatabases,
{
databases: [
{
dbDir: datasetUri.fsPath,
workingSet: 'default'
}
]
},
mockCancel,
mockProgress
);
});
it('should not register if unsupported', async () => {
const qs = createMockQueryServerClient(
{
cliConstraints: {
supportsDatabaseRegistration: () => false
}
} as any);
const runner = new LegacyQueryRunner(qs);
const mockProgress = 'progress-monitor';
const mockCancel = 'cancel-token';
const datasetUri = Uri.file('dataset-uri');
const dbItem: DatabaseItem = {
contents: {
datasetUri
}
} as any;
await runner.registerDatabase(mockProgress as any, mockCancel as any, dbItem);
expect(qs.sendRequest).not.to.have.been.called;
});
it('should not deregister if unsupported', async () => {
const qs = createMockQueryServerClient(
{
cliConstraints: {
supportsDatabaseRegistration: () => false
}
} as any);
const runner = new LegacyQueryRunner(qs);
const mockProgress = 'progress-monitor';
const mockCancel = 'cancel-token';
const datasetUri = Uri.file('dataset-uri');
const dbItem: DatabaseItem = {
contents: {
datasetUri
}
} as any;
await runner.registerDatabase(mockProgress as any, mockCancel as any, dbItem);
expect(qs.sendRequest).not.to.have.been.called;
});
});
let queryNum = 0;
function createMockQueryInfo(databaseHasMetadataFile = true, saveDir = `save-dir${queryNum++}`) {
return new QueryEvaluationInfo(
return new QueryInProgress(
saveDir,
Uri.parse('file:///abc').fsPath,
databaseHasMetadataFile,

12
extensions/ql-vscode/test/pure-tests/variant-analysis.test.ts Normal file
View File

@@ -0,0 +1,12 @@
import { expect } from 'chai';
import { parseVariantAnalysisQueryLanguage, VariantAnalysisQueryLanguage } from '../../src/remote-queries/shared/variant-analysis';
describe('parseVariantAnalysisQueryLanguage', () => {
it('parses a valid language', () => {
expect(parseVariantAnalysisQueryLanguage('javascript')).to.equal(VariantAnalysisQueryLanguage.Javascript);
});
it('returns undefined for an valid language', () => {
expect(parseVariantAnalysisQueryLanguage('rubbish')).to.not.exist;
});
});