diff --git a/extensions/ql-vscode/src/cli.ts b/extensions/ql-vscode/src/cli.ts index 1594c6e65..20710bca7 100644 --- a/extensions/ql-vscode/src/cli.ts +++ b/extensions/ql-vscode/src/cli.ts @@ -16,7 +16,7 @@ import { DistributionProvider, FindDistributionResultKind } from './distribution import { assertNever, getErrorMessage, getErrorStack } from './pure/helpers-pure'; import { QueryMetadata, SortDirection } from './pure/interface-types'; import { Logger, ProgressReporter } from './logging'; -import { CompilationMessage } from './pure/messages'; +import { CompilationMessage } from './pure/legacy-messages'; import { sarifParser } from './sarif-parser'; import { dbSchemeToLanguage, walkDirectory } from './helpers'; @@ -1248,6 +1248,9 @@ export class CliVersionConstraint { */ public static CLI_VERSION_WITH_LANGUAGE = new SemVer('2.4.1'); + + public static CLI_VERSION_WITH_NONDESTURCTIVE_UPGRADES = new SemVer('2.4.2'); + /** * CLI version where `codeql resolve upgrades` supports * the `--allow-downgrades` flag @@ -1264,14 +1267,6 @@ export class CliVersionConstraint { */ public static CLI_VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1'); - /** - * CLI version where non destructive upgrades were introduced. - * - * This was landed in multiple parts so this is the version where all necessary feature were supported. - */ - public static CLI_VERSION_WITH_NON_DESTRUCTIVE_UPGRADES = new SemVer('2.4.2'); - - /** * CLI version where the `--allow-library-packs` option to `codeql resolve queries` was * introduced. @@ -1351,6 +1346,10 @@ export class CliVersionConstraint { return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_LANGUAGE); } + public async supportsNonDestructiveUpgrades() { + return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NONDESTURCTIVE_UPGRADES); + } + public async supportsDowngrades() { return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DOWNGRADES); } @@ -1367,10 +1366,6 @@ export class CliVersionConstraint { return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DB_REGISTRATION); } - async supportsNonDestructiveUpgrades(): Promise { - return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NON_DESTRUCTIVE_UPGRADES); - } - async supportsDatabaseUnbundle() { return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DATABASE_UNBUNDLE); } diff --git a/extensions/ql-vscode/src/contextual/astBuilder.ts b/extensions/ql-vscode/src/contextual/astBuilder.ts index 88adb6f9d..39b9d8938 100644 --- a/extensions/ql-vscode/src/contextual/astBuilder.ts +++ b/extensions/ql-vscode/src/contextual/astBuilder.ts @@ -1,10 +1,10 @@ -import { QueryWithResults } from '../run-queries'; import { CodeQLCliServer } from '../cli'; import { DecodedBqrsChunk, BqrsId, EntityValue } from '../pure/bqrs-cli-types'; import { DatabaseItem } from '../databases'; import { ChildAstItem, AstItem } from '../astViewer'; import fileRangeFromURI from './fileRangeFromURI'; import { Uri } from 'vscode'; +import { QueryWithResults } from '../run-queries-shared'; /** * A class that wraps a tree of QL results from a query that diff --git a/extensions/ql-vscode/src/contextual/locationFinder.ts b/extensions/ql-vscode/src/contextual/locationFinder.ts index 6c95b6406..553c0d710 100644 --- a/extensions/ql-vscode/src/contextual/locationFinder.ts +++ b/extensions/ql-vscode/src/contextual/locationFinder.ts @@ -3,13 +3,12 @@ import { ColumnKindCode, EntityValue, getResultSetSchema, ResultSetSchema } from import { CodeQLCliServer } from '../cli'; import { DatabaseManager, DatabaseItem } from '../databases'; import fileRangeFromURI from './fileRangeFromURI'; -import * as messages from '../pure/messages'; -import { QueryServerClient } from '../queryserver-client'; -import { QueryWithResults, compileAndRunQueryAgainstDatabase, createInitialQueryInfo } from '../run-queries'; import { ProgressCallback } from '../commandRunner'; import { KeyType } from './keyType'; import { qlpackOfDatabase, resolveQueries } from './queryResolver'; import { CancellationToken, LocationLink, Uri } from 'vscode'; +import { createInitialQueryInfo, QueryWithResults } from '../run-queries-shared'; +import { QueryRunner } from '../queryRunner'; export const SELECT_QUERY_NAME = '#select'; export const TEMPLATE_NAME = 'selectedSourceFile'; @@ -35,7 +34,7 @@ export interface FullLocationLink extends LocationLink { */ export async function getLocationsForUriString( cli: CodeQLCliServer, - qs: QueryServerClient, + qs: QueryRunner, dbm: DatabaseManager, uriString: string, keyType: KeyType, @@ -65,19 +64,8 @@ export async function getLocationsForUriString( }, false ); - - const results = await compileAndRunQueryAgainstDatabase( - cli, - qs, - db, - initialInfo, - queryStorageDir, - progress, - token, - templates - ); - - if (results.result.resultType == messages.QueryResultType.SUCCESS) { + const results = await qs.compileAndRunQueryAgainstDatabase(db, initialInfo, queryStorageDir, progress, token, templates); + if (results.sucessful) { links.push(...await getLinksFromResults(results, cli, db, filter)); } } @@ -114,15 +102,9 @@ async function getLinksFromResults( return localLinks; } -function createTemplates(path: string): messages.TemplateDefinitions { +function createTemplates(path: string): Record { return { - [TEMPLATE_NAME]: { - values: { - tuples: [[{ - stringValue: path - }]] - } - } + [TEMPLATE_NAME]: path }; } diff --git a/extensions/ql-vscode/src/contextual/templateProvider.ts b/extensions/ql-vscode/src/contextual/templateProvider.ts index 356dd7500..4af7f35cb 100644 --- a/extensions/ql-vscode/src/contextual/templateProvider.ts +++ b/extensions/ql-vscode/src/contextual/templateProvider.ts @@ -16,9 +16,6 @@ import { CodeQLCliServer } from '../cli'; import { DatabaseManager } from '../databases'; import { CachedOperation } from '../helpers'; import { ProgressCallback, withProgress } from '../commandRunner'; -import * as messages from '../pure/messages'; -import { QueryServerClient } from '../queryserver-client'; -import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo, QueryWithResults } from '../run-queries'; import AstBuilder from './astBuilder'; import { KeyType, @@ -26,6 +23,8 @@ import { import { FullLocationLink, getLocationsForUriString, TEMPLATE_NAME } from './locationFinder'; import { qlpackOfDatabase, resolveQueries } from './queryResolver'; import { isCanary, NO_CACHE_AST_VIEWER } from '../config'; +import { createInitialQueryInfo, QueryWithResults } from '../run-queries-shared'; +import { QueryRunner } from '../queryRunner'; /** * Run templated CodeQL queries to find definitions and references in @@ -39,7 +38,7 @@ export class TemplateQueryDefinitionProvider implements DefinitionProvider { constructor( private cli: CodeQLCliServer, - private qs: QueryServerClient, + private qs: QueryRunner, private dbm: DatabaseManager, private queryStorageDir: string, ) { @@ -83,7 +82,7 @@ export class TemplateQueryReferenceProvider implements ReferenceProvider { constructor( private cli: CodeQLCliServer, - private qs: QueryServerClient, + private qs: QueryRunner, private dbm: DatabaseManager, private queryStorageDir: string, ) { @@ -137,7 +136,7 @@ export class TemplatePrintAstProvider { constructor( private cli: CodeQLCliServer, - private qs: QueryServerClient, + private qs: QueryRunner, private dbm: DatabaseManager, private queryStorageDir: string, ) { @@ -195,14 +194,9 @@ export class TemplatePrintAstProvider { } const query = queries[0]; - const templates: messages.TemplateDefinitions = { - [TEMPLATE_NAME]: { - values: { - tuples: [[{ - stringValue: zippedArchive.pathWithinSourceArchive - }]] - } - } + const templates: Record = { + [TEMPLATE_NAME]: + zippedArchive.pathWithinSourceArchive }; const initialInfo = await createInitialQueryInfo( @@ -215,9 +209,7 @@ export class TemplatePrintAstProvider { ); return { - query: await compileAndRunQueryAgainstDatabase( - this.cli, - this.qs, + query: await this.qs.compileAndRunQueryAgainstDatabase( db, initialInfo, this.queryStorageDir, @@ -231,23 +223,23 @@ export class TemplatePrintAstProvider { } export class TemplatePrintCfgProvider { - private cache: CachedOperation<[Uri, messages.TemplateDefinitions] | undefined>; + private cache: CachedOperation<[Uri, Record] | undefined>; constructor( private cli: CodeQLCliServer, private dbm: DatabaseManager, ) { - this.cache = new CachedOperation<[Uri, messages.TemplateDefinitions] | undefined>(this.getCfgUri.bind(this)); + this.cache = new CachedOperation<[Uri, Record] | undefined>(this.getCfgUri.bind(this)); } - async provideCfgUri(document?: TextDocument): Promise<[Uri, messages.TemplateDefinitions] | undefined> { + async provideCfgUri(document?: TextDocument): Promise<[Uri, Record] | undefined> { if (!document) { return; } return await this.cache.get(document.uri.toString()); } - private async getCfgUri(uriString: string): Promise<[Uri, messages.TemplateDefinitions]> { + private async getCfgUri(uriString: string): Promise<[Uri, Record]> { const uri = Uri.parse(uriString, true); if (uri.scheme !== zipArchiveScheme) { throw new Error('CFG Viewing is only available for databases with zipped source archives.'); @@ -275,14 +267,8 @@ export class TemplatePrintCfgProvider { const queryUri = Uri.file(queries[0]); - const templates: messages.TemplateDefinitions = { - [TEMPLATE_NAME]: { - values: { - tuples: [[{ - stringValue: zippedArchive.pathWithinSourceArchive - }]] - } - } + const templates: Record = { + [TEMPLATE_NAME]: zippedArchive.pathWithinSourceArchive }; return [queryUri, templates]; diff --git a/extensions/ql-vscode/src/databases-ui.ts b/extensions/ql-vscode/src/databases-ui.ts index bea2bc425..b33cc0a2a 100644 --- a/extensions/ql-vscode/src/databases-ui.ts +++ b/extensions/ql-vscode/src/databases-ui.ts @@ -28,9 +28,6 @@ import { showAndLogErrorMessage } from './helpers'; import { logger } from './logging'; -import { clearCacheInDatabase } from './run-queries'; -import * as qsClient from './queryserver-client'; -import { upgradeDatabaseExplicit } from './upgrades'; import { importArchiveDatabase, promptImportGithubDatabase, @@ -40,6 +37,7 @@ import { import { CancellationToken } from 'vscode'; import { asyncFilter, getErrorMessage } from './pure/helpers-pure'; import { Credentials } from './authentication'; +import { QueryRunner } from './queryRunner'; import { isCanary } from './config'; type ThemableIconPath = { light: string; dark: string } | string; @@ -220,7 +218,7 @@ export class DatabaseUI extends DisposableObject { public constructor( private databaseManager: DatabaseManager, - private readonly queryServer: qsClient.QueryServerClient | undefined, + private readonly queryServer: QueryRunner | undefined, private readonly storagePath: string, readonly extensionPath: string, private readonly getCredentials: () => Promise @@ -390,9 +388,9 @@ export class DatabaseUI extends DisposableObject { handleChooseDatabaseFolder = async ( progress: ProgressCallback, token: CancellationToken - ): Promise => { + ): Promise => { try { - return await this.chooseAndSetDatabase(true, progress, token); + await this.chooseAndSetDatabase(true, progress, token); } catch (e) { void showAndLogErrorMessage(getErrorMessage(e)); return undefined; @@ -458,9 +456,9 @@ export class DatabaseUI extends DisposableObject { handleChooseDatabaseArchive = async ( progress: ProgressCallback, token: CancellationToken - ): Promise => { + ): Promise => { try { - return await this.chooseAndSetDatabase(false, progress, token); + await this.chooseAndSetDatabase(false, progress, token); } catch (e) { void showAndLogErrorMessage(getErrorMessage(e)); return undefined; @@ -576,8 +574,7 @@ export class DatabaseUI extends DisposableObject { // Search for upgrade scripts in any workspace folders available - await upgradeDatabaseExplicit( - this.queryServer, + await this.queryServer.upgradeDatabaseExplicit( databaseItem, progress, token @@ -592,8 +589,7 @@ export class DatabaseUI extends DisposableObject { this.queryServer !== undefined && this.databaseManager.currentDatabaseItem !== undefined ) { - await clearCacheInDatabase( - this.queryServer, + await this.queryServer.clearCacheInDatabase( this.databaseManager.currentDatabaseItem, progress, token diff --git a/extensions/ql-vscode/src/databases.ts b/extensions/ql-vscode/src/databases.ts index a1c7ed170..79af4fc3e 100644 --- a/extensions/ql-vscode/src/databases.ts +++ b/extensions/ql-vscode/src/databases.ts @@ -17,9 +17,8 @@ import { import { zipArchiveScheme, encodeArchiveBasePath, decodeSourceArchiveUri, encodeSourceArchiveUri } from './archive-filesystem-provider'; import { DisposableObject } from './pure/disposable-object'; import { Logger, logger } from './logging'; -import { registerDatabases, Dataset, deregisterDatabases } from './pure/messages'; -import { QueryServerClient } from './queryserver-client'; import { getErrorMessage } from './pure/helpers-pure'; +import { QueryRunner } from './queryRunner'; /** * databases.ts @@ -555,13 +554,13 @@ export class DatabaseManager extends DisposableObject { constructor( private readonly ctx: ExtensionContext, - private readonly qs: QueryServerClient, + private readonly qs: QueryRunner, private readonly cli: cli.CodeQLCliServer, public logger: Logger ) { super(); - qs.onDidStartQueryServer(this.reregisterDatabases.bind(this)); + qs.onStart(this.reregisterDatabases.bind(this)); // Let this run async. void this.loadPersistedState(); @@ -860,27 +859,14 @@ export class DatabaseManager extends DisposableObject { token: vscode.CancellationToken, dbItem: DatabaseItem, ) { - if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) { - const databases: Dataset[] = [{ - dbDir: dbItem.contents.datasetUri.fsPath, - workingSet: 'default' - }]; - await this.qs.sendRequest(deregisterDatabases, { databases }, token, progress); - } + await this.qs.deregisterDatabase(progress, token, dbItem); } - private async registerDatabase( progress: ProgressCallback, token: vscode.CancellationToken, dbItem: DatabaseItem, ) { - if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) { - const databases: Dataset[] = [{ - dbDir: dbItem.contents.datasetUri.fsPath, - workingSet: 'default' - }]; - await this.qs.sendRequest(registerDatabases, { databases }, token, progress); - } + await this.qs.registerDatabase(progress, token, dbItem); } private updatePersistedCurrentDatabaseItem(): void { diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index 0bc919e79..da9a284eb 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -70,12 +70,11 @@ import { asError, assertNever, getErrorMessage } from './pure/helpers-pure'; import { spawnIdeServer } from './ide-server'; import { InterfaceManager } from './interface'; import { WebviewReveal } from './interface-utils'; -import { ideServerLogger, logger, queryServerLogger } from './logging'; +import { ideServerLogger, logger, ProgressReporter, queryServerLogger } from './logging'; import { QueryHistoryManager } from './query-history'; import { CompletedLocalQueryInfo, LocalQueryInfo } from './query-results'; -import * as qsClient from './queryserver-client'; +import * as qsClient from './legacy-query-server/queryserver-client'; import { displayQuickQuery } from './quick-query'; -import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo } from './run-queries'; import { QLTestAdapterFactory } from './test-adapter'; import { TestUIService } from './test-ui'; import { CompareInterfaceManager } from './compare/compare-interface'; @@ -102,6 +101,9 @@ import { EvalLogViewer } from './eval-log-viewer'; import { SummaryLanguageSupport } from './log-insights/summary-language-support'; import { JoinOrderScannerProvider } from './log-insights/join-order'; import { LogScannerService } from './log-insights/log-scanner-service'; +import { createInitialQueryInfo } from './run-queries-shared'; +import { LegacyQueryRunner } from './legacy-query-server/legacyRunner'; +import { QueryRunner } from './queryRunner'; /** * extension.ts @@ -164,7 +166,7 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command: export interface CodeQLExtensionInterface { readonly ctx: ExtensionContext; readonly cliServer: CodeQLCliServer; - readonly qs: qsClient.QueryServerClient; + readonly qs: QueryRunner; readonly distributionManager: DistributionManager; readonly databaseManager: DatabaseManager; readonly databaseUI: DatabaseUI; @@ -416,21 +418,7 @@ async function activateWithInstalledDistribution( ctx.subscriptions.push(statusBar); void logger.log('Initializing query server client.'); - const qs = new qsClient.QueryServerClient( - qlConfigurationListener, - cliServer, - { - logger: queryServerLogger, - contextStoragePath: getContextStoragePath(ctx), - }, - (task) => - Window.withProgress( - { title: 'CodeQL query server', location: ProgressLocation.Window }, - task - ) - ); - ctx.subscriptions.push(qs); - await qs.startQueryServer(); + const qs = await createQueryServer(qlConfigurationListener, cliServer, ctx); void logger.log('Initializing database manager.'); const dbm = new DatabaseManager(ctx, qs, cliServer, logger); @@ -552,9 +540,7 @@ async function activateWithInstalledDistribution( const item = new LocalQueryInfo(initialInfo, source); qhm.addQuery(item); try { - const completedQueryInfo = await compileAndRunQueryAgainstDatabase( - cliServer, - qs, + const completedQueryInfo = await qs.compileAndRunQueryAgainstDatabase( databaseItem, initialInfo, queryStorageDir, @@ -788,6 +774,7 @@ async function activateWithInstalledDistribution( // requests for each query to run. // Only do it if running multiple queries since this check is // performed on each query run anyway. + // Don't do this with non destructive upgrades as the user won't see anything anyway. await databaseUI.tryUpgradeCurrentDatabase(progress, token); } @@ -1139,6 +1126,26 @@ async function activateWithInstalledDistribution( }; } +async function createQueryServer(qlConfigurationListener: QueryServerConfigListener, cliServer: CodeQLCliServer, ctx: ExtensionContext): Promise { + const qsOpts = { + logger: queryServerLogger, + contextStoragePath: getContextStoragePath(ctx), + }; + const progressCallback = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable) => Window.withProgress( + { title: 'CodeQL query server', location: ProgressLocation.Window }, + task + ); + const qs = new qsClient.QueryServerClient( + qlConfigurationListener, + cliServer, + qsOpts, + progressCallback + ); + ctx.subscriptions.push(qs); + await qs.startQueryServer(); + return new LegacyQueryRunner(qs); +} + function getContextStoragePath(ctx: ExtensionContext) { return ctx.storageUri?.fsPath || ctx.globalStorageUri.fsPath; } diff --git a/extensions/ql-vscode/src/interface.ts b/extensions/ql-vscode/src/interface.ts index efc8f85e5..da1eee50d 100644 --- a/extensions/ql-vscode/src/interface.ts +++ b/extensions/ql-vscode/src/interface.ts @@ -29,10 +29,9 @@ import { RawResultsSortState, } from './pure/interface-types'; import { Logger } from './logging'; -import * as messages from './pure/messages'; import { commandRunner } from './commandRunner'; import { CompletedQueryInfo, interpretResultsSarif, interpretGraphResults } from './query-results'; -import { QueryEvaluationInfo } from './run-queries'; +import { QueryEvaluationInfo } from './run-queries-shared'; import { parseSarifLocation, parseSarifPlainTextMessage } from './pure/sarif-utils'; import { WebviewReveal, @@ -327,7 +326,7 @@ export class InterfaceManager extends AbstractInterfaceManager { - if (fullQuery.completedQuery.result.resultType !== messages.QueryResultType.SUCCESS) { + if (!fullQuery.completedQuery.sucessful) { return; } diff --git a/extensions/ql-vscode/src/json-rpc-server.ts b/extensions/ql-vscode/src/json-rpc-server.ts new file mode 100644 index 000000000..12d7e810a --- /dev/null +++ b/extensions/ql-vscode/src/json-rpc-server.ts @@ -0,0 +1,30 @@ +import { Logger } from './logging'; +import * as cp from 'child_process'; +import { Disposable } from 'vscode'; +import { MessageConnection } from 'vscode-jsonrpc'; + + +/** A running query server process and its associated message connection. */ +export class ServerProcess implements Disposable { + child: cp.ChildProcess; + connection: MessageConnection; + logger: Logger; + + constructor(child: cp.ChildProcess, connection: MessageConnection, logger: Logger) { + this.child = child; + this.connection = connection; + this.logger = logger; + } + + dispose(): void { + void this.logger.log('Stopping query server...'); + this.connection.dispose(); + this.child.stdin!.end(); + this.child.stderr!.destroy(); + // TODO kill the process if it doesn't terminate after a certain time limit. + + // On Windows, we usually have to terminate the process before closing its stdout. + this.child.stdout!.destroy(); + void this.logger.log('Stopped query server.'); + } +} diff --git a/extensions/ql-vscode/src/legacy-query-server/legacyRunner.ts b/extensions/ql-vscode/src/legacy-query-server/legacyRunner.ts new file mode 100644 index 000000000..43a7d8311 --- /dev/null +++ b/extensions/ql-vscode/src/legacy-query-server/legacyRunner.ts @@ -0,0 +1,59 @@ +import { CancellationToken } from 'vscode'; +import { ProgressCallback } from '../commandRunner'; +import { DatabaseItem } from '../databases'; +import { Dataset, deregisterDatabases, registerDatabases } from '../pure/legacy-messages'; +import { InitialQueryInfo, LocalQueryInfo } from '../query-results'; +import { QueryRunner } from '../queryRunner'; +import { QueryWithResults } from '../run-queries-shared'; +import { QueryServerClient } from './queryserver-client'; +import { clearCacheInDatabase, compileAndRunQueryAgainstDatabase } from './run-queries'; +import { upgradeDatabaseExplicit } from './upgrades'; + +export class LegacyQueryRunner extends QueryRunner { + + + constructor(public readonly qs: QueryServerClient) { + super(); + } + + get cliServer() { + return this.qs.cliServer; + } + + async restartQueryServer(progress: ProgressCallback, token: CancellationToken): Promise { + await this.qs.restartQueryServer(progress, token); + } + + onStart(callBack: (progress: ProgressCallback, token: CancellationToken) => Promise) { + this.qs.onDidStartQueryServer(callBack); + } + async clearCacheInDatabase(dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken): Promise { + await clearCacheInDatabase(this.qs, dbItem, progress, token); + } + async compileAndRunQueryAgainstDatabase(dbItem: DatabaseItem, initialInfo: InitialQueryInfo, queryStorageDir: string, progress: ProgressCallback, token: CancellationToken, templates?: Record, queryInfo?: LocalQueryInfo): Promise { + return await compileAndRunQueryAgainstDatabase(this.qs.cliServer, this.qs, dbItem, initialInfo, queryStorageDir, progress, token, templates, queryInfo); + } + + async deregisterDatabase(progress: ProgressCallback, token: CancellationToken, dbItem: DatabaseItem): Promise { + if (dbItem.contents && (await this.qs.cliServer.cliConstraints.supportsDatabaseRegistration())) { + const databases: Dataset[] = [{ + dbDir: dbItem.contents.datasetUri.fsPath, + workingSet: 'default' + }]; + await this.qs.sendRequest(deregisterDatabases, { databases }, token, progress); + } + } + async registerDatabase(progress: ProgressCallback, token: CancellationToken, dbItem: DatabaseItem): Promise { + if (dbItem.contents && (await this.qs.cliServer.cliConstraints.supportsDatabaseRegistration())) { + const databases: Dataset[] = [{ + dbDir: dbItem.contents.datasetUri.fsPath, + workingSet: 'default' + }]; + await this.qs.sendRequest(registerDatabases, { databases }, token, progress); + } + } + + async upgradeDatabaseExplicit(dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken): Promise { + await upgradeDatabaseExplicit(this.qs, dbItem, progress, token); + } +} diff --git a/extensions/ql-vscode/src/queryserver-client.ts b/extensions/ql-vscode/src/legacy-query-server/queryserver-client.ts similarity index 78% rename from extensions/ql-vscode/src/queryserver-client.ts rename to extensions/ql-vscode/src/legacy-query-server/queryserver-client.ts index 277b46b2d..f77fef2dd 100644 --- a/extensions/ql-vscode/src/queryserver-client.ts +++ b/extensions/ql-vscode/src/legacy-query-server/queryserver-client.ts @@ -1,49 +1,25 @@ -import * as cp from 'child_process'; import * as path from 'path'; import * as fs from 'fs-extra'; -import { DisposableObject } from './pure/disposable-object'; -import { Disposable, CancellationToken, commands } from 'vscode'; -import { createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc'; -import * as cli from './cli'; -import { QueryServerConfig } from './config'; -import { Logger, ProgressReporter } from './logging'; -import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from './pure/messages'; -import * as messages from './pure/messages'; -import { ProgressCallback, ProgressTask } from './commandRunner'; +import { DisposableObject } from '../pure/disposable-object'; +import { CancellationToken, commands } from 'vscode'; +import { createMessageConnection, RequestType } from 'vscode-jsonrpc'; +import * as cli from '../cli'; +import { QueryServerConfig } from '../config'; +import { Logger, ProgressReporter } from '../logging'; +import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from '../pure/legacy-messages'; +import * as messages from '../pure/legacy-messages'; +import { ProgressCallback, ProgressTask } from '../commandRunner'; +import { findQueryLogFile } from '../run-queries-shared'; +import { ServerProcess } from '../json-rpc-server'; + +type WithProgressReporting = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable) => Thenable; type ServerOpts = { logger: Logger; contextStoragePath: string; } -/** A running query server process and its associated message connection. */ -class ServerProcess implements Disposable { - child: cp.ChildProcess; - connection: MessageConnection; - logger: Logger; - - constructor(child: cp.ChildProcess, connection: MessageConnection, logger: Logger) { - this.child = child; - this.connection = connection; - this.logger = logger; - } - - dispose(): void { - void this.logger.log('Stopping query server...'); - this.connection.dispose(); - this.child.stdin!.end(); - this.child.stderr!.destroy(); - // TODO kill the process if it doesn't terminate after a certain time limit. - - // On Windows, we usually have to terminate the process before closing its stdout. - this.child.stdout!.destroy(); - void this.logger.log('Stopped query server.'); - } -} - -type WithProgressReporting = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable) => Thenable; - /** * Client that manages a query server process. * The server process is started upon initialization and tracked during its lifetime. @@ -254,27 +230,3 @@ export class QueryServerClient extends DisposableObject { } } } - -export function findQueryLogFile(resultPath: string): string { - return path.join(resultPath, 'query.log'); -} - -export function findQueryEvalLogFile(resultPath: string): string { - return path.join(resultPath, 'evaluator-log.jsonl'); -} - -export function findQueryEvalLogSummaryFile(resultPath: string): string { - return path.join(resultPath, 'evaluator-log.summary'); -} - -export function findJsonQueryEvalLogSummaryFile(resultPath: string): string { - return path.join(resultPath, 'evaluator-log.summary.jsonl'); -} - -export function findQueryEvalLogSummarySymbolsFile(resultPath: string): string { - return path.join(resultPath, 'evaluator-log.summary.symbols.json'); -} - -export function findQueryEvalLogEndSummaryFile(resultPath: string): string { - return path.join(resultPath, 'evaluator-log-end.summary'); -} diff --git a/extensions/ql-vscode/src/legacy-query-server/run-queries.ts b/extensions/ql-vscode/src/legacy-query-server/run-queries.ts new file mode 100644 index 000000000..c6b2f5cde --- /dev/null +++ b/extensions/ql-vscode/src/legacy-query-server/run-queries.ts @@ -0,0 +1,518 @@ +import * as crypto from 'crypto'; +import * as fs from 'fs-extra'; +import * as tmp from 'tmp-promise'; +import * as path from 'path'; +import { + CancellationToken, + Uri, +} from 'vscode'; +import { ErrorCodes, ResponseError } from 'vscode-languageclient'; + +import * as cli from '../cli'; +import { DatabaseItem, } from '../databases'; +import { + getOnDiskWorkspaceFolders, + showAndLogErrorMessage, + showAndLogWarningMessage, + tryGetQueryMetadata, + upgradesTmpDir +} from '../helpers'; +import { ProgressCallback } from '../commandRunner'; +import { QueryMetadata } from '../pure/interface-types'; +import { logger } from '../logging'; +import * as messages from '../pure/legacy-messages'; +import { InitialQueryInfo, LocalQueryInfo } from '../query-results'; +import * as qsClient from './queryserver-client'; +import { getErrorMessage } from '../pure/helpers-pure'; +import { compileDatabaseUpgradeSequence, upgradeDatabaseExplicit } from './upgrades'; +import { QueryEvaluationInfo, QueryWithResults } from '../run-queries-shared'; + +/** + * A collection of evaluation-time information about a query, + * including the query itself, and where we have decided to put + * temporary files associated with it, such as the compiled query + * output and results. + */ +export class QueryInProgress { + + public queryEvalInfo: QueryEvaluationInfo; + /** + * Note that in the {@link FullQueryInfo.slurp} method, we create a QueryEvaluationInfo instance + * by explicitly setting the prototype in order to avoid calling this constructor. + */ + constructor( + readonly querySaveDir: string, + readonly dbItemPath: string, + databaseHasMetadataFile: boolean, + readonly queryDbscheme: string, // the dbscheme file the query expects, based on library path resolution + readonly quickEvalPosition?: messages.Position, + readonly metadata?: QueryMetadata, + readonly templates?: Record, + ) { + this.queryEvalInfo = new QueryEvaluationInfo(querySaveDir, dbItemPath, databaseHasMetadataFile, quickEvalPosition, metadata); + /**/ + } + + get compiledQueryPath() { + return path.join(this.querySaveDir, 'compiledQuery.qlo'); + } + + + async run( + qs: qsClient.QueryServerClient, + upgradeQlo: string | undefined, + availableMlModels: cli.MlModelInfo[], + dbItem: DatabaseItem, + progress: ProgressCallback, + token: CancellationToken, + queryInfo?: LocalQueryInfo, + ): Promise { + if (!dbItem.contents || dbItem.error) { + throw new Error('Can\'t run query on invalid database.'); + } + + let result: messages.EvaluationResult | null = null; + + const callbackId = qs.registerCallback(res => { + result = { + ...res, + logFileLocation: this.queryEvalInfo.logPath + }; + }); + + const availableMlModelUris: messages.MlModel[] = availableMlModels.map(model => ({ uri: Uri.file(model.path).toString(true) })); + + const queryToRun: messages.QueryToRun = { + resultsPath: this.queryEvalInfo.resultsPaths.resultsPath, + qlo: Uri.file(this.compiledQueryPath).toString(), + compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(), + allowUnknownTemplates: true, + templateValues: createSimpleTemplates(this.templates), + availableMlModels: availableMlModelUris, + id: callbackId, + timeoutSecs: qs.config.timeoutSecs, + }; + + const dataset: messages.Dataset = { + dbDir: dbItem.contents.datasetUri.fsPath, + workingSet: 'default' + }; + if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) { + await qs.sendRequest(messages.startLog, { + db: dataset, + logPath: this.queryEvalInfo.evalLogPath, + }); + + } + const params: messages.EvaluateQueriesParams = { + db: dataset, + evaluateId: callbackId, + queries: [queryToRun], + stopOnError: false, + useSequenceHint: false + }; + try { + await qs.sendRequest(messages.runQueries, params, token, progress); + if (qs.config.customLogDirectory) { + void showAndLogWarningMessage( + `Custom log directories are no longer supported. The "codeQL.runningQueries.customLogDirectory" setting is deprecated. Unset the setting to stop seeing this message. Query logs saved to ${this.queryEvalInfo.logPath}.` + ); + } + } finally { + qs.unRegisterCallback(callbackId); + if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) { + await qs.sendRequest(messages.endLog, { + db: dataset, + logPath: this.queryEvalInfo.evalLogPath, + }); + if (await this.queryEvalInfo.hasEvalLog()) { + await this.queryEvalInfo.addQueryLogs(queryInfo, qs.cliServer, qs.logger); + } else { + void showAndLogWarningMessage(`Failed to write structured evaluator log to ${this.queryEvalInfo.evalLogPath}.`); + } + } + } + return result || { + evaluationTime: 0, + message: 'No result from server', + queryId: -1, + runId: callbackId, + resultType: messages.QueryResultType.OTHER_ERROR + }; + } + + async compile( + qs: qsClient.QueryServerClient, + program: messages.QlProgram, + progress: ProgressCallback, + token: CancellationToken, + ): Promise { + let compiled: messages.CheckQueryResult | undefined; + try { + const target = this.quickEvalPosition ? { + quickEval: { quickEvalPos: this.quickEvalPosition } + } : { query: {} }; + const params: messages.CompileQueryParams = { + compilationOptions: { + computeNoLocationUrls: true, + failOnWarnings: false, + fastCompilation: false, + includeDilInQlo: true, + localChecking: false, + noComputeGetUrl: false, + noComputeToString: false, + computeDefaultStrings: true, + emitDebugInfo: true + }, + extraOptions: { + timeoutSecs: qs.config.timeoutSecs + }, + queryToCheck: program, + resultPath: this.compiledQueryPath, + target, + }; + + compiled = await qs.sendRequest(messages.compileQuery, params, token, progress); + } finally { + void qs.logger.log(' - - - COMPILATION DONE - - - ', { additionalLogLocation: this.queryEvalInfo.logPath }); + } + return (compiled?.messages || []).filter(msg => msg.severity === messages.Severity.ERROR); + } +} + +export async function clearCacheInDatabase( + qs: qsClient.QueryServerClient, + dbItem: DatabaseItem, + progress: ProgressCallback, + token: CancellationToken, +): Promise { + if (dbItem.contents === undefined) { + throw new Error('Can\'t clear the cache in an invalid database.'); + } + + const db: messages.Dataset = { + dbDir: dbItem.contents.datasetUri.fsPath, + workingSet: 'default', + }; + + const params: messages.ClearCacheParams = { + dryRun: false, + db, + }; + + return qs.sendRequest(messages.clearCache, params, token, progress); +} + + +/** + * Compare the dbscheme implied by the query `query` and that of the current database. + * - If they are compatible, do nothing. + * - If they are incompatible but the database can be upgraded, suggest that upgrade. + * - If they are incompatible and the database cannot be upgraded, throw an error. + */ +async function checkDbschemeCompatibility( + cliServer: cli.CodeQLCliServer, + qs: qsClient.QueryServerClient, + query: QueryInProgress, + qlProgram: messages.QlProgram, + dbItem: DatabaseItem, + progress: ProgressCallback, + token: CancellationToken, +): Promise { + const searchPath = getOnDiskWorkspaceFolders(); + + if (dbItem.contents?.dbSchemeUri !== undefined) { + const { finalDbscheme } = await cliServer.resolveUpgrades(dbItem.contents.dbSchemeUri.fsPath, searchPath, false); + const hash = async function(filename: string): Promise { + return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex'); + }; + + // At this point, we have learned about three dbschemes: + + // the dbscheme of the actual database we're querying. + const dbschemeOfDb = await hash(dbItem.contents.dbSchemeUri.fsPath); + + // the dbscheme of the query we're running, including the library we've resolved it to use. + const dbschemeOfLib = await hash(query.queryDbscheme); + + // the database we're able to upgrade to + const upgradableTo = await hash(finalDbscheme); + + if (upgradableTo != dbschemeOfLib) { + reportNoUpgradePath(qlProgram, query); + } + + if (upgradableTo == dbschemeOfLib && + dbschemeOfDb != dbschemeOfLib) { + // Try to upgrade the database + await upgradeDatabaseExplicit( + qs, + dbItem, + progress, + token + ); + } + } +} + +function reportNoUpgradePath(qlProgram: messages.QlProgram, query: QueryInProgress): void { + throw new Error( + `Query ${qlProgram.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.` + ); +} + +/** + * Compile a non-destructive upgrade. + */ +async function compileNonDestructiveUpgrade( + qs: qsClient.QueryServerClient, + upgradeTemp: tmp.DirectoryResult, + query: QueryInProgress, + qlProgram: messages.QlProgram, + dbItem: DatabaseItem, + progress: ProgressCallback, + token: CancellationToken, +): Promise { + + if (!dbItem?.contents?.dbSchemeUri) { + throw new Error('Database is invalid, and cannot be upgraded.'); + } + + // When packaging is used, dependencies may exist outside of the workspace and they are always on the resolved search path. + // When packaging is not used, all dependencies are in the workspace. + const upgradesPath = (await qs.cliServer.cliConstraints.supportsPackaging()) + ? qlProgram.libraryPath + : getOnDiskWorkspaceFolders(); + + const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades( + dbItem.contents.dbSchemeUri.fsPath, + upgradesPath, + true, + query.queryDbscheme + ); + + if (!matchesTarget) { + reportNoUpgradePath(qlProgram, query); + } + const result = await compileDatabaseUpgradeSequence(qs, dbItem, scripts, upgradeTemp, progress, token); + if (result.compiledUpgrade === undefined) { + const error = result.error || '[no error message available]'; + throw new Error(error); + } + // We can upgrade to the actual target + qlProgram.dbschemePath = query.queryDbscheme; + // We are new enough that we will always support single file upgrades. + return result.compiledUpgrade; +} + + + +export async function compileAndRunQueryAgainstDatabase( + cliServer: cli.CodeQLCliServer, + qs: qsClient.QueryServerClient, + dbItem: DatabaseItem, + initialInfo: InitialQueryInfo, + queryStorageDir: string, + progress: ProgressCallback, + token: CancellationToken, + templates?: Record, + queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query. +): Promise { + if (!dbItem.contents || !dbItem.contents.dbSchemeUri) { + throw new Error(`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`); + } + + // Get the workspace folder paths. + const diskWorkspaceFolders = getOnDiskWorkspaceFolders(); + // Figure out the library path for the query. + const packConfig = await cliServer.resolveLibraryPath(diskWorkspaceFolders, initialInfo.queryPath); + + if (!packConfig.dbscheme) { + throw new Error('Could not find a database scheme for this query. Please check that you have a valid qlpack.yml file for this query, which refers to a database scheme either in the `dbscheme` field or through one of its dependencies.'); + } + + // Check whether the query has an entirely different schema from the + // database. (Queries that merely need the database to be upgraded + // won't trigger this check) + // This test will produce confusing results if we ever change the name of the database schema files. + const querySchemaName = path.basename(packConfig.dbscheme); + const dbSchemaName = path.basename(dbItem.contents.dbSchemeUri.fsPath); + if (querySchemaName != dbSchemaName) { + void logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`); + throw new Error(`The query ${path.basename(initialInfo.queryPath)} cannot be run against the selected database (${dbItem.name}): their target languages are different. Please select a different database and try again.`); + } + + const qlProgram: messages.QlProgram = { + // The project of the current document determines which library path + // we use. The `libraryPath` field in this server message is relative + // to the workspace root, not to the project root. + libraryPath: packConfig.libraryPath, + // Since we are compiling and running a query against a database, + // we use the database's DB scheme here instead of the DB scheme + // from the current document's project. + dbschemePath: dbItem.contents.dbSchemeUri.fsPath, + queryPath: initialInfo.queryPath + }; + + // Read the query metadata if possible, to use in the UI. + const metadata = await tryGetQueryMetadata(cliServer, qlProgram.queryPath); + + let availableMlModels: cli.MlModelInfo[] = []; + if (!await cliServer.cliConstraints.supportsResolveMlModels()) { + void logger.log('Resolving ML models is unsupported by this version of the CLI. Running the query without any ML models.'); + } else { + try { + availableMlModels = (await cliServer.resolveMlModels(diskWorkspaceFolders, initialInfo.queryPath)).models; + if (availableMlModels.length) { + void logger.log(`Found available ML models at the following paths: ${availableMlModels.map(x => `'${x.path}'`).join(', ')}.`); + } else { + void logger.log('Did not find any available ML models.'); + } + } catch (e) { + const message = `Couldn't resolve available ML models for ${qlProgram.queryPath}. Running the ` + + `query without any ML models: ${e}.`; + void showAndLogErrorMessage(message); + } + } + + const hasMetadataFile = (await dbItem.hasMetadataFile()); + const query = new QueryInProgress( + path.join(queryStorageDir, initialInfo.id), + dbItem.databaseUri.fsPath, + hasMetadataFile, + packConfig.dbscheme, + initialInfo.quickEvalPosition, + metadata, + templates + ); + await query.queryEvalInfo.createTimestampFile(); + + let upgradeDir: tmp.DirectoryResult | undefined; + try { + let upgradeQlo; + if (await cliServer.cliConstraints.supportsNonDestructiveUpgrades()) { + upgradeDir = await tmp.dir({ dir: upgradesTmpDir, unsafeCleanup: true }); + upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, qlProgram, dbItem, progress, token); + } else { + await checkDbschemeCompatibility(cliServer, qs, query, qlProgram, dbItem, progress, token); + } + let errors; + try { + errors = await query.compile(qs, qlProgram, progress, token); + } catch (e) { + if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) { + return createSyntheticResult(query, 'Query cancelled'); + } else { + throw e; + } + } + + if (errors.length === 0) { + const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token, queryInfo); + if (result.resultType !== messages.QueryResultType.SUCCESS) { + const message = result.message || 'Failed to run query'; + void logger.log(message); + void showAndLogErrorMessage(message); + } + const message = formatLegacyMessage(result); + + return { + query: query.queryEvalInfo, + message: message, + sucessful: result.resultType == messages.QueryResultType.SUCCESS, + logFileLocation: result.logFileLocation, + dispose: () => { + qs.logger.removeAdditionalLogLocation(result.logFileLocation); + } + }; + } else { + // Error dialogs are limited in size and scrollability, + // so we include a general description of the problem, + // and direct the user to the output window for the detailed compilation messages. + // However we don't show quick eval errors there so we need to display them anyway. + void qs.logger.log( + `Failed to compile query ${initialInfo.queryPath} against database scheme ${qlProgram.dbschemePath}:`, + { additionalLogLocation: query.queryEvalInfo.logPath } + ); + + const formattedMessages: string[] = []; + + for (const error of errors) { + const message = error.message || '[no error message available]'; + const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`; + formattedMessages.push(formatted); + void qs.logger.log(formatted, { additionalLogLocation: query.queryEvalInfo.logPath }); + } + if (initialInfo.isQuickEval && formattedMessages.length <= 2) { + // If there are more than 2 error messages, they will not be displayed well in a popup + // and will be trimmed by the function displaying the error popup. Accordingly, we only + // try to show the errors if there are 2 or less, otherwise we direct the user to the log. + void showAndLogErrorMessage('Quick evaluation compilation failed: ' + formattedMessages.join('\n')); + } else { + void showAndLogErrorMessage((initialInfo.isQuickEval ? 'Quick evaluation' : 'Query') + compilationFailedErrorTail); + } + return createSyntheticResult(query, 'Query had compilation errors'); + } + } finally { + try { + await upgradeDir?.cleanup(); + } catch (e) { + void qs.logger.log( + `Could not clean up the upgrades dir. Reason: ${getErrorMessage(e)}`, + { additionalLogLocation: query.queryEvalInfo.logPath } + ); + } + } +} + + +const compilationFailedErrorTail = ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' + + ' and the query and database use the same target language. For more details on the error, go to View > Output,' + + ' and choose CodeQL Query Server from the dropdown.'; + +export function formatLegacyMessage(result: messages.EvaluationResult) { + switch (result.resultType) { + case messages.QueryResultType.CANCELLATION: + return `cancelled after ${Math.round(result.evaluationTime / 1000)} seconds`; + case messages.QueryResultType.OOM: + return 'out of memory'; + case messages.QueryResultType.SUCCESS: + return `finished in ${Math.round(result.evaluationTime / 1000)} seconds`; + case messages.QueryResultType.TIMEOUT: + return `timed out after ${Math.round(result.evaluationTime / 1000)} seconds`; + case messages.QueryResultType.OTHER_ERROR: + default: + return result.message ? `failed: ${result.message}` : 'failed'; + } +} + +/** + * Create a synthetic result for a query that failed to compile. + */ +function createSyntheticResult( + query: QueryInProgress, + message: string, +): QueryWithResults { + return { + query: query.queryEvalInfo, + message, + sucessful: false, + dispose: () => { /**/ }, + }; +} + +function createSimpleTemplates(templates: Record | undefined): messages.TemplateDefinitions | undefined { + if (!templates) { + return undefined; + } + const result: messages.TemplateDefinitions = {}; + for (const key of Object.keys(templates)) { + result[key] = { + values: { + tuples: [[{ stringValue: templates[key] }]] + } + }; + } + return result; +} + diff --git a/extensions/ql-vscode/src/upgrades.ts b/extensions/ql-vscode/src/legacy-query-server/upgrades.ts similarity index 96% rename from extensions/ql-vscode/src/upgrades.ts rename to extensions/ql-vscode/src/legacy-query-server/upgrades.ts index a7427c10a..aecc6f0c5 100644 --- a/extensions/ql-vscode/src/upgrades.ts +++ b/extensions/ql-vscode/src/legacy-query-server/upgrades.ts @@ -1,12 +1,12 @@ import * as vscode from 'vscode'; -import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, tmpDir } from './helpers'; -import { ProgressCallback, UserCancellationException } from './commandRunner'; -import { logger } from './logging'; -import * as messages from './pure/messages'; -import * as qsClient from './queryserver-client'; +import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, tmpDir } from '../helpers'; +import { ProgressCallback, UserCancellationException } from '../commandRunner'; +import { logger } from '../logging'; +import * as messages from '../pure/legacy-messages'; +import * as qsClient from '../legacy-query-server/queryserver-client'; import * as tmp from 'tmp-promise'; import * as path from 'path'; -import { DatabaseItem } from './databases'; +import { DatabaseItem } from '../databases'; /** * Maximum number of lines to include from database upgrade message, @@ -16,7 +16,6 @@ import { DatabaseItem } from './databases'; const MAX_UPGRADE_MESSAGE_LINES = 10; - /** * Compile a database upgrade sequence. * Callers must check that this is valid with the current queryserver first. diff --git a/extensions/ql-vscode/src/pure/messages.ts b/extensions/ql-vscode/src/pure/legacy-messages.ts similarity index 93% rename from extensions/ql-vscode/src/pure/messages.ts rename to extensions/ql-vscode/src/pure/legacy-messages.ts index aeb691f86..b8ac43765 100644 --- a/extensions/ql-vscode/src/pure/messages.ts +++ b/extensions/ql-vscode/src/pure/legacy-messages.ts @@ -15,38 +15,7 @@ */ import * as rpc from 'vscode-jsonrpc'; - -/** - * A position within a QL file. - */ -export interface Position { - /** - * The one-based index of the start line - */ - line: number; - /** - * The one-based offset of the start column within - * the start line in UTF-16 code-units - */ - column: number; - /** - * The one-based index of the end line line - */ - endLine: number; - - /** - * The one-based offset of the end column within - * the end line in UTF-16 code-units - */ - endColumn: number; - /** - * The path of the file. - * If the file name is "Compiler Generated" the - * the position is not a real position but - * arises from compiler generated code. - */ - fileName: string; -} +import * as shared from './messages-shared'; /** * A query that should be checked for any errors or warnings @@ -258,28 +227,6 @@ export interface DILQuery { dilSource: string; } -/** - * The way of compiling the query, as a normal query - * or a subset of it. Note that precisely one of the two options should be set. - */ -export interface CompilationTarget { - /** - * Compile as a normal query - */ - query?: Record; - /** - * Compile as a quick evaluation - */ - quickEval?: QuickEvalOptions; -} - -/** - * Options for quick evaluation - */ -export interface QuickEvalOptions { - quickEvalPos?: Position; -} - /** * The result of checking a query. */ @@ -1012,37 +959,20 @@ export type DeregisterDatabasesResult = { }; /** - * Type for any action that could have progress messages. + * A position within a QL file. */ -export interface WithProgressId { - /** - * The main body - */ - body: T; - /** - * The id used to report progress updates - */ - progressId: number; -} +export type Position = shared.Position; -export interface ProgressMessage { - /** - * The id of the operation that is running - */ - id: number; - /** - * The current step - */ - step: number; - /** - * The maximum step. This *should* be constant for a single job. - */ - maxStep: number; - /** - * The current progress message - */ - message: string; -} +/** + * The way of compiling the query, as a normal query + * or a subset of it. Note that precisely one of the two options should be set. + */ +export type CompilationTarget = shared.CompilationTarget; + +export type QuickEvalOptions = shared.QuickEvalOptions; + +export type WithProgressId = shared.WithProgressId; +export type ProgressMessage = shared.ProgressMessage; /** * Check a Ql query for errors without compiling it @@ -1120,7 +1050,4 @@ export const deregisterDatabases = new rpc.RequestType< */ export const completeQuery = new rpc.RequestType, void, void>('evaluation/queryCompleted'); -/** - * A notification that the progress has been changed. - */ -export const progress = new rpc.NotificationType('ql/progressUpdated'); +export const progress = shared.progress; diff --git a/extensions/ql-vscode/src/pure/messages-shared.ts b/extensions/ql-vscode/src/pure/messages-shared.ts new file mode 100644 index 000000000..ee7b04240 --- /dev/null +++ b/extensions/ql-vscode/src/pure/messages-shared.ts @@ -0,0 +1,110 @@ +/** + * Types for messages exchanged during jsonrpc communication with the + * the CodeQL query server. + * + * This file exists in the queryserver and in the vscode extension, and + * should be kept in sync between them. + * + * A note about the namespaces below, which look like they are + * essentially enums, namely Severity, ResultColumnKind, and + * QueryResultType. By design, for the sake of extensibility, clients + * receiving messages of this protocol are supposed to accept any + * number for any of these types. We commit to the given meaning of + * the numbers listed in constants in the namespaces, and we commit to + * the fact that any unknown QueryResultType value counts as an error. + */ + +import * as rpc from 'vscode-jsonrpc'; + +/** + * A position within a QL file. + */ +export interface Position { + /** + * The one-based index of the start line + */ + line: number; + /** + * The one-based offset of the start column within + * the start line in UTF-16 code-units + */ + column: number; + /** + * The one-based index of the end line line + */ + endLine: number; + + /** + * The one-based offset of the end column within + * the end line in UTF-16 code-units + */ + endColumn: number; + /** + * The path of the file. + * If the file name is "Compiler Generated" the + * the position is not a real position but + * arises from compiler generated code. + */ + fileName: string; +} + +/** + * The way of compiling the query, as a normal query + * or a subset of it. Note that precisely one of the two options should be set. + */ +export interface CompilationTarget { + /** + * Compile as a normal query + */ + query?: Record; + /** + * Compile as a quick evaluation + */ + quickEval?: QuickEvalOptions; +} + +/** + * Options for quick evaluation + */ +export interface QuickEvalOptions { + quickEvalPos?: Position; +} + +/** + * Type for any action that could have progress messages. + */ +export interface WithProgressId { + /** + * The main body + */ + body: T; + /** + * The id used to report progress updates + */ + progressId: number; +} + +export interface ProgressMessage { + /** + * The id of the operation that is running + */ + id: number; + /** + * The current step + */ + step: number; + /** + * The maximum step. This *should* be constant for a single job. + */ + maxStep: number; + /** + * The current progress message + */ + message: string; +} + + +/** + * A notification that the progress has been changed. + */ +export const progress = new rpc.NotificationType('ql/progressUpdated'); diff --git a/extensions/ql-vscode/src/query-history.ts b/extensions/ql-vscode/src/query-history.ts index f12452c07..2b4ea3b4b 100644 --- a/extensions/ql-vscode/src/query-history.ts +++ b/extensions/ql-vscode/src/query-history.ts @@ -26,7 +26,6 @@ import { } from './helpers'; import { logger } from './logging'; import { URLSearchParams } from 'url'; -import { QueryServerClient } from './queryserver-client'; import { DisposableObject } from './pure/disposable-object'; import { commandRunner } from './commandRunner'; import { ONE_HOUR_IN_MS, TWO_HOURS_IN_MS } from './pure/time'; @@ -48,7 +47,8 @@ import { WebviewReveal } from './interface-utils'; import { EvalLogViewer } from './eval-log-viewer'; import EvalLogTreeBuilder from './eval-log-tree-builder'; import { EvalLogData, parseViewerData } from './pure/log-summary-parser'; -import { QueryWithResults } from './run-queries'; +import { QueryWithResults } from './run-queries-shared'; +import { QueryRunner } from './queryRunner'; /** * query-history.ts @@ -329,7 +329,7 @@ export class QueryHistoryManager extends DisposableObject { readonly onDidCompleteQuery = this._onDidCompleteQuery.event; constructor( - private readonly qs: QueryServerClient, + private readonly qs: QueryRunner, private readonly dbm: DatabaseManager, private readonly localQueriesInterfaceManager: InterfaceManager, private readonly remoteQueriesManager: RemoteQueriesManager, @@ -791,7 +791,7 @@ export class QueryHistoryManager extends DisposableObject { throw new Error('Please select a local query.'); } - if (!finalSingleItem.completedQuery?.didRunSuccessfully) { + if (!finalSingleItem.completedQuery?.sucessful) { throw new Error('Please select a query that has completed successfully.'); } @@ -1072,7 +1072,7 @@ export class QueryHistoryManager extends DisposableObject { void this.tryOpenExternalFile(query.csvPath); return; } - if (await query.exportCsvResults(this.qs, query.csvPath)) { + if (await query.exportCsvResults(this.qs.cliServer, query.csvPath)) { void this.tryOpenExternalFile( query.csvPath ); @@ -1091,7 +1091,7 @@ export class QueryHistoryManager extends DisposableObject { } await this.tryOpenExternalFile( - await finalSingleItem.completedQuery.query.ensureCsvAlerts(this.qs, this.dbm) + await finalSingleItem.completedQuery.query.ensureCsvAlerts(this.qs.cliServer, this.dbm) ); } @@ -1107,7 +1107,7 @@ export class QueryHistoryManager extends DisposableObject { } await this.tryOpenExternalFile( - await finalSingleItem.completedQuery.query.ensureDilPath(this.qs) + await finalSingleItem.completedQuery.query.ensureDilPath(this.qs.cliServer) ); } @@ -1232,7 +1232,7 @@ the file in the file explorer and dragging it into the workspace.` if (!otherQuery.completedQuery) { throw new Error('Please select a completed query.'); } - if (!otherQuery.completedQuery.didRunSuccessfully) { + if (!otherQuery.completedQuery.sucessful) { throw new Error('Please select a successful query.'); } if (otherQuery.initialInfo.databaseInfo.name !== dbName) { @@ -1252,7 +1252,7 @@ the file in the file explorer and dragging it into the workspace.` otherQuery !== singleItem && otherQuery.t === 'local' && otherQuery.completedQuery && - otherQuery.completedQuery.didRunSuccessfully && + otherQuery.completedQuery.sucessful && otherQuery.initialInfo.databaseInfo.name === dbName ) .map((item) => ({ diff --git a/extensions/ql-vscode/src/query-results.ts b/extensions/ql-vscode/src/query-results.ts index 536426901..dd49d8b29 100644 --- a/extensions/ql-vscode/src/query-results.ts +++ b/extensions/ql-vscode/src/query-results.ts @@ -1,7 +1,7 @@ import { CancellationTokenSource, env } from 'vscode'; -import { QueryWithResults, QueryEvaluationInfo } from './run-queries'; -import * as messages from './pure/messages'; +import * as messages from './pure/messages-shared'; +import * as legacyMessages from './pure/legacy-messages'; import * as cli from './cli'; import * as fs from 'fs-extra'; import * as path from 'path'; @@ -17,6 +17,8 @@ import { import { DatabaseInfo } from './pure/interface-types'; import { QueryStatus } from './query-status'; import { RemoteQueryHistoryItem } from './remote-queries/remote-query-history-item'; +import { QueryEvaluationInfo, QueryWithResults } from './run-queries-shared'; +import { formatLegacyMessage } from './legacy-query-server/run-queries'; /** * query-results.ts @@ -44,7 +46,12 @@ export interface InitialQueryInfo { export class CompletedQueryInfo implements QueryWithResults { readonly query: QueryEvaluationInfo; - readonly result: messages.EvaluationResult; + readonly message?: string; + readonly sucessful?: boolean; + /** + * The legacy result. This is only set when loading from the query history. + */ + readonly result?: legacyMessages.EvaluationResult; readonly logFileLocation?: string; resultCount: number; @@ -75,9 +82,9 @@ export class CompletedQueryInfo implements QueryWithResults { evaluation: QueryWithResults, ) { this.query = evaluation.query; - this.result = evaluation.result; this.logFileLocation = evaluation.logFileLocation; - + this.message = evaluation.message; + this.sucessful = evaluation.sucessful; // Use the dispose method from the evaluation. // The dispose will clean up any additional log locations that this // query may have created. @@ -92,18 +99,12 @@ export class CompletedQueryInfo implements QueryWithResults { } get statusString(): string { - switch (this.result.resultType) { - case messages.QueryResultType.CANCELLATION: - return `cancelled after ${Math.round(this.result.evaluationTime / 1000)} seconds`; - case messages.QueryResultType.OOM: - return 'out of memory'; - case messages.QueryResultType.SUCCESS: - return `finished in ${Math.round(this.result.evaluationTime / 1000)} seconds`; - case messages.QueryResultType.TIMEOUT: - return `timed out after ${Math.round(this.result.evaluationTime / 1000)} seconds`; - case messages.QueryResultType.OTHER_ERROR: - default: - return this.result.message ? `failed: ${this.result.message}` : 'failed'; + if (this.message) { + return this.message; + } else if (this.result) { + return formatLegacyMessage(this.result); + } else { + throw new Error('No status available'); } } @@ -115,10 +116,6 @@ export class CompletedQueryInfo implements QueryWithResults { || this.query.resultsPaths.resultsPath; } - get didRunSuccessfully(): boolean { - return this.result.resultType === messages.QueryResultType.SUCCESS; - } - async updateSortState( server: cli.CodeQLCliServer, resultSetName: string, @@ -302,7 +299,7 @@ export class LocalQueryInfo { return QueryStatus.Failed; } else if (!this.completedQuery) { return QueryStatus.InProgress; - } else if (this.completedQuery.didRunSuccessfully) { + } else if (this.completedQuery.sucessful) { return QueryStatus.Completed; } else { return QueryStatus.Failed; diff --git a/extensions/ql-vscode/src/query-serialization.ts b/extensions/ql-vscode/src/query-serialization.ts index e27100594..521e02f78 100644 --- a/extensions/ql-vscode/src/query-serialization.ts +++ b/extensions/ql-vscode/src/query-serialization.ts @@ -5,7 +5,7 @@ import { showAndLogErrorMessage } from './helpers'; import { asyncFilter, getErrorMessage, getErrorStack } from './pure/helpers-pure'; import { CompletedQueryInfo, LocalQueryInfo, QueryHistoryInfo } from './query-results'; import { QueryStatus } from './query-status'; -import { QueryEvaluationInfo } from './run-queries'; +import { QueryEvaluationInfo } from './run-queries-shared'; export async function slurpQueryHistory(fsPath: string): Promise { try { diff --git a/extensions/ql-vscode/src/queryRunner.ts b/extensions/ql-vscode/src/queryRunner.ts new file mode 100644 index 000000000..307fc0516 --- /dev/null +++ b/extensions/ql-vscode/src/queryRunner.ts @@ -0,0 +1,48 @@ +import { CancellationToken } from 'vscode'; +import { CodeQLCliServer } from './cli'; +import { ProgressCallback } from './commandRunner'; +import { DatabaseItem } from './databases'; +import { InitialQueryInfo, LocalQueryInfo } from './query-results'; +import { QueryWithResults } from './run-queries-shared'; + + + +export abstract class QueryRunner { + abstract restartQueryServer(progress: ProgressCallback, token: CancellationToken): Promise; + + abstract cliServer: CodeQLCliServer; + + abstract onStart(arg0: (progress: ProgressCallback, token: CancellationToken) => Promise): void; + abstract clearCacheInDatabase( + dbItem: DatabaseItem, + progress: ProgressCallback, + token: CancellationToken): Promise; + + abstract compileAndRunQueryAgainstDatabase( + dbItem: DatabaseItem, + initialInfo: InitialQueryInfo, + queryStorageDir: string, + progress: ProgressCallback, + token: CancellationToken, + templates?: Record, + queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query. + ): Promise; + + abstract deregisterDatabase( + progress: ProgressCallback, + token: CancellationToken, + dbItem: DatabaseItem, + ): Promise; + + abstract registerDatabase( + progress: ProgressCallback, + token: CancellationToken, + dbItem: DatabaseItem, + ): Promise; + + abstract upgradeDatabaseExplicit( + dbItem: DatabaseItem, + progress: ProgressCallback, + token: CancellationToken, + ): Promise +} diff --git a/extensions/ql-vscode/src/run-queries-shared.ts b/extensions/ql-vscode/src/run-queries-shared.ts new file mode 100644 index 000000000..377813d6c --- /dev/null +++ b/extensions/ql-vscode/src/run-queries-shared.ts @@ -0,0 +1,585 @@ +import * as messages from './pure/messages-shared'; +import * as legacyMessages from './pure/legacy-messages'; +import { DatabaseInfo, QueryMetadata } from './pure/interface-types'; +import * as path from 'path'; +import { createTimestampFile, showAndLogWarningMessage } from './helpers'; +import { + ConfigurationTarget, + Range, + TextDocument, + TextEditor, + Uri, + window +} from 'vscode'; +import * as config from './config'; +import { UserCancellationException } from './commandRunner'; +import * as fs from 'fs-extra'; +import { ensureMetadataIsComplete, InitialQueryInfo, LocalQueryInfo } from './query-results'; +import { isQuickQueryPath } from './quick-query'; +import { nanoid } from 'nanoid'; +import { CodeQLCliServer } from './cli'; +import { SELECT_QUERY_NAME } from './contextual/locationFinder'; +import { DatabaseManager } from './databases'; +import { DecodedBqrsChunk } from './pure/bqrs-cli-types'; +import { logger, Logger } from './logging'; +import { generateSummarySymbolsFile } from './log-insights/summary-parser'; +import { asError } from './pure/helpers-pure'; + + + +/** + * run-queries.ts + * -------------- + * + * Compiling and running QL queries. + */ + +export function findQueryLogFile(resultPath: string): string { + return path.join(resultPath, 'query.log'); +} + +function findQueryEvalLogFile(resultPath: string): string { + return path.join(resultPath, 'evaluator-log.jsonl'); +} + +function findQueryEvalLogSummaryFile(resultPath: string): string { + return path.join(resultPath, 'evaluator-log.summary'); +} + +function findJsonQueryEvalLogSummaryFile(resultPath: string): string { + return path.join(resultPath, 'evaluator-log.summary.jsonl'); +} + +function findQueryEvalLogSummarySymbolsFile(resultPath: string): string { + return path.join(resultPath, 'evaluator-log.summary.symbols.json'); +} + +function findQueryEvalLogEndSummaryFile(resultPath: string): string { + return path.join(resultPath, 'evaluator-log-end.summary'); +} + + +export class QueryEvaluationInfo { + + /** + * Note that in the {@link FullQueryInfo.slurp} method, we create a QueryEvaluationInfo instance + * by explicitly setting the prototype in order to avoid calling this constructor. + */ + constructor( + public readonly querySaveDir: string, + public readonly dbItemPath: string, + private readonly databaseHasMetadataFile: boolean, + public readonly quickEvalPosition?: messages.Position, + public readonly metadata?: QueryMetadata, + ) { + /**/ + } + + get dilPath() { + return path.join(this.querySaveDir, 'results.dil'); + } + + /** + * Get the path that the compiled query is if it exists. Note that it only exists when using the legacy query server. + */ + get compileQueryPath() { + return path.join(this.querySaveDir, 'compiledQuery.qlo'); + } + + get csvPath() { + return path.join(this.querySaveDir, 'results.csv'); + } + + get logPath() { + return findQueryLogFile(this.querySaveDir); + } + + get evalLogPath() { + return findQueryEvalLogFile(this.querySaveDir); + } + + get evalLogSummaryPath() { + return findQueryEvalLogSummaryFile(this.querySaveDir); + } + + get jsonEvalLogSummaryPath() { + return findJsonQueryEvalLogSummaryFile(this.querySaveDir); + } + + get evalLogSummarySymbolsPath() { + return findQueryEvalLogSummarySymbolsFile(this.querySaveDir); + } + + get evalLogEndSummaryPath() { + return findQueryEvalLogEndSummaryFile(this.querySaveDir); + } + + get resultsPaths() { + return { + resultsPath: path.join(this.querySaveDir, 'results.bqrs'), + interpretedResultsPath: path.join(this.querySaveDir, + this.metadata?.kind === 'graph' + ? 'graphResults' + : 'interpretedResults.sarif' + ), + }; + } + getSortedResultSetPath(resultSetName: string) { + return path.join(this.querySaveDir, `sortedResults-${resultSetName}.bqrs`); + } + + /** + * Creates a file in the query directory that indicates when this query was created. + * This is important for keeping track of when queries should be removed. + */ + async createTimestampFile() { + await createTimestampFile(this.querySaveDir); + } + + + /** + * Holds if this query can in principle produce interpreted results. + */ + canHaveInterpretedResults(): boolean { + if (!this.databaseHasMetadataFile) { + void logger.log('Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.'); + return false; + } + + const kind = this.metadata?.kind; + const hasKind = !!kind; + if (!hasKind) { + void logger.log('Cannot produce interpreted results since the query does not have @kind metadata.'); + return false; + } + + // Graph queries only return interpreted results if we are in canary mode. + if (kind === 'graph') { + return config.isCanary(); + } + + // table is the default query kind. It does not produce interpreted results. + // any query kind that is not table can, in principle, produce interpreted results. + return kind !== 'table'; + } + + /** + * Holds if this query actually has produced interpreted results. + */ + async hasInterpretedResults(): Promise { + return fs.pathExists(this.resultsPaths.interpretedResultsPath); + } + + /** + * Holds if this query already has DIL produced + */ + async hasDil(): Promise { + return fs.pathExists(this.dilPath); + } + + /** + * Holds if this query already has CSV results produced + */ + async hasCsv(): Promise { + return fs.pathExists(this.csvPath); + } + + /** + * Returns the path to the DIL file produced by this query. If the query has not yet produced DIL, + * this will return first create the DIL file and then return the path to the DIL file. + */ + async ensureDilPath(cliServer: CodeQLCliServer): Promise { + if (await this.hasDil()) { + return this.dilPath; + } + const compiledQuery = path.join(this.querySaveDir, 'compiledQuery.qlo'); + if (!(await fs.pathExists(compiledQuery))) { + throw new Error( + `Cannot create DIL because compiled query is missing. ${compiledQuery}` + ); + } + + await cliServer.generateDil(compiledQuery, this.dilPath); + return this.dilPath; + } + + /** + * Holds if this query already has a completed structured evaluator log + */ + async hasEvalLog(): Promise { + return fs.pathExists(this.evalLogPath); + } + + async addQueryLogs(queryInfo: LocalQueryInfo, cliServer: CodeQLCliServer, logger: Logger) { + queryInfo.evalLogLocation = this.evalLogPath; + queryInfo.evalLogSummaryLocation = await this.generateHumanReadableLogSummary(cliServer); + void this.logEndSummary(queryInfo.evalLogSummaryLocation, logger); // Logged asynchrnously + if (config.isCanary()) { // Generate JSON summary for viewer. + await cliServer.generateJsonLogSummary(this.evalLogPath, this.jsonEvalLogSummaryPath); + queryInfo.jsonEvalLogSummaryLocation = this.jsonEvalLogSummaryPath; + await generateSummarySymbolsFile(this.evalLogSummaryPath, this.evalLogSummarySymbolsPath); + queryInfo.evalLogSummarySymbolsLocation = this.evalLogSummarySymbolsPath; + } + } + + /** + * Calls the appropriate CLI command to generate a human-readable log summary. + * @param qs The query server client. + * @returns The path to the log summary, or `undefined` if the summary could not be generated. */ + private async generateHumanReadableLogSummary(cliServer: CodeQLCliServer): Promise { + try { + await cliServer.generateLogSummary(this.evalLogPath, this.evalLogSummaryPath, this.evalLogEndSummaryPath); + return this.evalLogSummaryPath; + } catch (e) { + const err = asError(e); + void showAndLogWarningMessage(`Failed to generate human-readable structured evaluator log summary. Reason: ${err.message}`); + return undefined; + } + } + + /** + * Logs the end summary to the Output window and log file. + * @param logSummaryPath Path to the human-readable log summary + * @param qs The query server client. + */ + private async logEndSummary(logSummaryPath: string | undefined, logger: Logger): Promise { + if (logSummaryPath === undefined) { + // Failed to generate the log, so we don't expect an end summary either. + return; + } + + try { + const endSummaryContent = await fs.readFile(this.evalLogEndSummaryPath, 'utf-8'); + void logger.log(' --- Evaluator Log Summary --- ', { additionalLogLocation: this.logPath }); + void logger.log(endSummaryContent, { additionalLogLocation: this.logPath }); + } catch (e) { + void showAndLogWarningMessage(`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`); + } + } + + /** + * Creates the CSV file containing the results of this query. This will only be called if the query + * does not have interpreted results and the CSV file does not already exist. + * + * @return Promise if the operation creates the file. Promise if the operation does + * not create the file. + * + * @throws Error if the operation fails. + */ + async exportCsvResults(cliServer: CodeQLCliServer, csvPath: string): Promise { + const resultSet = await this.chooseResultSet(cliServer); + if (!resultSet) { + void showAndLogWarningMessage('Query has no result set.'); + return false; + } + let stopDecoding = false; + const out = fs.createWriteStream(csvPath); + + const promise: Promise = new Promise((resolve, reject) => { + out.on('finish', () => resolve(true)); + out.on('error', () => { + if (!stopDecoding) { + stopDecoding = true; + reject(new Error(`Failed to write CSV results to ${csvPath}`)); + } + }); + }); + + let nextOffset: number | undefined = 0; + do { + const chunk: DecodedBqrsChunk = await cliServer.bqrsDecode(this.resultsPaths.resultsPath, resultSet, { + pageSize: 100, + offset: nextOffset, + }); + chunk.tuples.forEach((tuple) => { + out.write(tuple.map((v, i) => + chunk.columns[i].kind === 'String' + ? `"${typeof v === 'string' ? v.replaceAll('"', '""') : v}"` + : v + ).join(',') + '\n'); + }); + nextOffset = chunk.next; + } while (nextOffset && !stopDecoding); + out.end(); + + return promise; + } + + /** + * Choose the name of the result set to run. If the `#select` set exists, use that. Otherwise, + * arbitrarily choose the first set. Most of the time, this will be correct. + * + * If the query has no result sets, then return undefined. + */ + async chooseResultSet(cliServer: CodeQLCliServer) { + const resultSets = (await cliServer.bqrsInfo(this.resultsPaths.resultsPath, 0))['result-sets']; + if (!resultSets.length) { + return undefined; + } + if (resultSets.find(r => r.name === SELECT_QUERY_NAME)) { + return SELECT_QUERY_NAME; + } + return resultSets[0].name; + } + /** + * Returns the path to the CSV alerts interpretation of this query results. If CSV results have + * not yet been produced, this will return first create the CSV results and then return the path. + * + * This method only works for queries with interpreted results. + */ + async ensureCsvAlerts(cliServer: CodeQLCliServer, dbm: DatabaseManager): Promise { + if (await this.hasCsv()) { + return this.csvPath; + } + + const dbItem = dbm.findDatabaseItem(Uri.file(this.dbItemPath)); + if (!dbItem) { + throw new Error(`Cannot produce CSV results because database is missing. ${this.dbItemPath}`); + } + + let sourceInfo; + if (dbItem.sourceArchive !== undefined) { + sourceInfo = { + sourceArchive: dbItem.sourceArchive.fsPath, + sourceLocationPrefix: await dbItem.getSourceLocationPrefix( + cliServer + ), + }; + } + await cliServer.generateResultsCsv(ensureMetadataIsComplete(this.metadata), this.resultsPaths.resultsPath, this.csvPath, sourceInfo); + return this.csvPath; + } + + /** + * Cleans this query's results directory. + */ + async deleteQuery(): Promise { + await fs.remove(this.querySaveDir); + } +} + +export interface QueryWithResults { + readonly query: QueryEvaluationInfo; + readonly logFileLocation?: string; + readonly dispose: () => void; + readonly sucessful?: boolean; + readonly message?: string; + readonly result?: legacyMessages.EvaluationResult +} + + +/** + * Information about which query will be to be run. `quickEvalPosition` and `quickEvalText` + * is only filled in if the query is a quick query. + */ +interface SelectedQuery { + queryPath: string; + quickEvalPosition?: messages.Position; + quickEvalText?: string; +} + + +/** + * Determines which QL file to run during an invocation of `Run Query` or `Quick Evaluation`, as follows: + * - If the command was called by clicking on a file, then use that file. + * - Otherwise, use the file open in the current editor. + * - In either case, prompt the user to save the file if it is open with unsaved changes. + * - For `Quick Evaluation`, ensure the selected file is also the one open in the editor, + * and use the selected region. + * @param selectedResourceUri The selected resource when the command was run. + * @param quickEval Whether the command being run is `Quick Evaluation`. +*/ +export async function determineSelectedQuery( + selectedResourceUri: Uri | undefined, + quickEval: boolean, + range?: Range +): Promise { + const editor = window.activeTextEditor; + + // Choose which QL file to use. + let queryUri: Uri; + if (selectedResourceUri) { + // A resource was passed to the command handler, so use it. + queryUri = selectedResourceUri; + } else { + // No resource was passed to the command handler, so obtain it from the active editor. + // This usually happens when the command is called from the Command Palette. + if (editor === undefined) { + throw new Error('No query was selected. Please select a query and try again.'); + } else { + queryUri = editor.document.uri; + } + } + + if (queryUri.scheme !== 'file') { + throw new Error('Can only run queries that are on disk.'); + } + const queryPath = queryUri.fsPath; + + if (quickEval) { + if (!(queryPath.endsWith('.ql') || queryPath.endsWith('.qll'))) { + throw new Error('The selected resource is not a CodeQL file; It should have the extension ".ql" or ".qll".'); + } + } else { + if (!(queryPath.endsWith('.ql'))) { + throw new Error('The selected resource is not a CodeQL query file; It should have the extension ".ql".'); + } + } + + // Whether we chose the file from the active editor or from a context menu, + // if the same file is open with unsaved changes in the active editor, + // then prompt the user to save it first. + if (editor !== undefined && editor.document.uri.fsPath === queryPath) { + if (await promptUserToSaveChanges(editor.document)) { + await editor.document.save(); + } + } + + let quickEvalPosition: messages.Position | undefined = undefined; + let quickEvalText: string | undefined = undefined; + if (quickEval) { + if (editor == undefined) { + throw new Error('Can\'t run quick evaluation without an active editor.'); + } + if (editor.document.fileName !== queryPath) { + // For Quick Evaluation we expect these to be the same. + // Report an error if we end up in this (hopefully unlikely) situation. + throw new Error('The selected resource for quick evaluation should match the active editor.'); + } + quickEvalPosition = await getSelectedPosition(editor, range); + if (!editor.selection?.isEmpty) { + quickEvalText = editor.document.getText(editor.selection); + } else { + // capture the entire line if the user didn't select anything + const line = editor.document.lineAt(editor.selection.active.line); + quickEvalText = line.text.trim(); + } + } + + return { queryPath, quickEvalPosition, quickEvalText }; +} + + +/** Gets the selected position within the given editor. */ +async function getSelectedPosition(editor: TextEditor, range?: Range): Promise { + const selectedRange = range || editor.selection; + const pos = selectedRange.start; + const posEnd = selectedRange.end; + // Convert from 0-based to 1-based line and column numbers. + return { + fileName: await convertToQlPath(editor.document.fileName), + line: pos.line + 1, + column: pos.character + 1, + endLine: posEnd.line + 1, + endColumn: posEnd.character + 1 + }; +} + + +/** + * Prompts the user to save `document` if it has unsaved changes. + * + * @param document The document to save. + * + * @returns true if we should save changes and false if we should continue without saving changes. + * @throws UserCancellationException if we should abort whatever operation triggered this prompt + */ +async function promptUserToSaveChanges(document: TextDocument): Promise { + if (document.isDirty) { + if (config.AUTOSAVE_SETTING.getValue()) { + return true; + } + else { + const yesItem = { title: 'Yes', isCloseAffordance: false }; + const alwaysItem = { title: 'Always Save', isCloseAffordance: false }; + const noItem = { title: 'No (run version on disk)', isCloseAffordance: false }; + const cancelItem = { title: 'Cancel', isCloseAffordance: true }; + const message = 'Query file has unsaved changes. Save now?'; + const chosenItem = await window.showInformationMessage( + message, + { modal: true }, + yesItem, alwaysItem, noItem, cancelItem + ); + + if (chosenItem === alwaysItem) { + await config.AUTOSAVE_SETTING.updateValue(true, ConfigurationTarget.Workspace); + return true; + } + + if (chosenItem === yesItem) { + return true; + } + + if (chosenItem === cancelItem) { + throw new UserCancellationException('Query run cancelled.', true); + } + } + } + return false; +} + + +/** + * @param filePath This needs to be equivalent to Java's `Path.toRealPath(NO_FOLLOW_LINKS)` + */ +async function convertToQlPath(filePath: string): Promise { + if (process.platform === 'win32') { + + if (path.parse(filePath).root === filePath) { + // Java assumes uppercase drive letters are canonical. + return filePath.toUpperCase(); + } else { + const dir = await convertToQlPath(path.dirname(filePath)); + const fileName = path.basename(filePath); + const fileNames = await fs.readdir(dir); + for (const name of fileNames) { + // Leave the locale argument empty so that the default OS locale is used. + // We do this because this operation works on filesystem entities, which + // use the os locale, regardless of the locale of the running VS Code instance. + if (fileName.localeCompare(name, undefined, { sensitivity: 'accent' }) === 0) { + return path.join(dir, name); + } + } + } + throw new Error('Can\'t convert path to form suitable for QL:' + filePath); + } else { + return filePath; + } +} + + +/** + * Determines the initial information for a query. This is everything of interest + * we know about this query that is available before it is run. + * + * @param selectedQueryUri The Uri of the document containing the query to be run. + * @param databaseInfo The database to run the query against. + * @param isQuickEval true if this is a quick evaluation. + * @param range the selection range of the query to be run. Only used if isQuickEval is true. + * @returns The initial information for the query to be run. + */ +export async function createInitialQueryInfo( + selectedQueryUri: Uri | undefined, + databaseInfo: DatabaseInfo, + isQuickEval: boolean, + range?: Range +): Promise { + // Determine which query to run, based on the selection and the active editor. + const { queryPath, quickEvalPosition, quickEvalText } = await determineSelectedQuery(selectedQueryUri, isQuickEval, range); + + return { + queryPath, + isQuickEval, + isQuickQuery: isQuickQueryPath(queryPath), + databaseInfo, + id: `${path.basename(queryPath)}-${nanoid()}`, + start: new Date(), + ... (isQuickEval ? { + queryText: quickEvalText!, // if this query is quick eval, it must have quick eval text + quickEvalPosition: quickEvalPosition + } : { + queryText: await fs.readFile(queryPath, 'utf8') + }) + }; +} + diff --git a/extensions/ql-vscode/src/run-queries.ts b/extensions/ql-vscode/src/run-queries.ts deleted file mode 100644 index a64393e35..000000000 --- a/extensions/ql-vscode/src/run-queries.ts +++ /dev/null @@ -1,997 +0,0 @@ -import * as crypto from 'crypto'; -import * as fs from 'fs-extra'; -import * as tmp from 'tmp-promise'; -import * as path from 'path'; -import { nanoid } from 'nanoid'; -import { - CancellationToken, - ConfigurationTarget, - Range, - TextDocument, - TextEditor, - Uri, - window -} from 'vscode'; -import { ErrorCodes, ResponseError } from 'vscode-languageclient'; - -import * as cli from './cli'; -import * as config from './config'; -import { DatabaseItem, DatabaseManager } from './databases'; -import { - createTimestampFile, - getOnDiskWorkspaceFolders, - showAndLogErrorMessage, - showAndLogWarningMessage, - tryGetQueryMetadata, - upgradesTmpDir -} from './helpers'; -import { ProgressCallback, UserCancellationException } from './commandRunner'; -import { DatabaseInfo, QueryMetadata } from './pure/interface-types'; -import { logger } from './logging'; -import * as messages from './pure/messages'; -import { InitialQueryInfo, LocalQueryInfo } from './query-results'; -import * as qsClient from './queryserver-client'; -import { isQuickQueryPath } from './quick-query'; -import { compileDatabaseUpgradeSequence, upgradeDatabaseExplicit } from './upgrades'; -import { ensureMetadataIsComplete } from './query-results'; -import { SELECT_QUERY_NAME } from './contextual/locationFinder'; -import { DecodedBqrsChunk } from './pure/bqrs-cli-types'; -import { asError, getErrorMessage } from './pure/helpers-pure'; -import { generateSummarySymbolsFile } from './log-insights/summary-parser'; - -/** - * run-queries.ts - * -------------- - * - * Compiling and running QL queries. - */ - -/** - * Information about which query will be to be run. `quickEvalPosition` and `quickEvalText` - * is only filled in if the query is a quick query. - */ -interface SelectedQuery { - queryPath: string; - quickEvalPosition?: messages.Position; - quickEvalText?: string; -} - -/** - * A collection of evaluation-time information about a query, - * including the query itself, and where we have decided to put - * temporary files associated with it, such as the compiled query - * output and results. - */ -export class QueryEvaluationInfo { - - /** - * Note that in the {@link FullQueryInfo.slurp} method, we create a QueryEvaluationInfo instance - * by explicitly setting the prototype in order to avoid calling this constructor. - */ - constructor( - public readonly querySaveDir: string, - public readonly dbItemPath: string, - private readonly databaseHasMetadataFile: boolean, - public readonly queryDbscheme: string, // the dbscheme file the query expects, based on library path resolution - public readonly quickEvalPosition?: messages.Position, - public readonly metadata?: QueryMetadata, - public readonly templates?: messages.TemplateDefinitions - ) { - /**/ - } - - get dilPath() { - return path.join(this.querySaveDir, 'results.dil'); - } - - get csvPath() { - return path.join(this.querySaveDir, 'results.csv'); - } - - get compiledQueryPath() { - return path.join(this.querySaveDir, 'compiledQuery.qlo'); - } - - get logPath() { - return qsClient.findQueryLogFile(this.querySaveDir); - } - - get evalLogPath() { - return qsClient.findQueryEvalLogFile(this.querySaveDir); - } - - get evalLogSummaryPath() { - return qsClient.findQueryEvalLogSummaryFile(this.querySaveDir); - } - - get jsonEvalLogSummaryPath() { - return qsClient.findJsonQueryEvalLogSummaryFile(this.querySaveDir); - } - - get evalLogSummarySymbolsPath() { - return qsClient.findQueryEvalLogSummarySymbolsFile(this.querySaveDir); - } - - get evalLogEndSummaryPath() { - return qsClient.findQueryEvalLogEndSummaryFile(this.querySaveDir); - } - - get resultsPaths() { - return { - resultsPath: path.join(this.querySaveDir, 'results.bqrs'), - interpretedResultsPath: path.join(this.querySaveDir, - this.metadata?.kind === 'graph' - ? 'graphResults' - : 'interpretedResults.sarif' - ), - }; - } - - getSortedResultSetPath(resultSetName: string) { - return path.join(this.querySaveDir, `sortedResults-${resultSetName}.bqrs`); - } - - /** - * Creates a file in the query directory that indicates when this query was created. - * This is important for keeping track of when queries should be removed. - */ - async createTimestampFile() { - await createTimestampFile(this.querySaveDir); - } - - async run( - qs: qsClient.QueryServerClient, - upgradeQlo: string | undefined, - availableMlModels: cli.MlModelInfo[], - dbItem: DatabaseItem, - progress: ProgressCallback, - token: CancellationToken, - queryInfo?: LocalQueryInfo, - ): Promise { - if (!dbItem.contents || dbItem.error) { - throw new Error('Can\'t run query on invalid database.'); - } - - let result: messages.EvaluationResult | null = null; - - const callbackId = qs.registerCallback(res => { - result = { - ...res, - logFileLocation: this.logPath - }; - }); - - const availableMlModelUris: messages.MlModel[] = availableMlModels.map(model => ({ uri: Uri.file(model.path).toString(true) })); - - const queryToRun: messages.QueryToRun = { - resultsPath: this.resultsPaths.resultsPath, - qlo: Uri.file(this.compiledQueryPath).toString(), - compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(), - allowUnknownTemplates: true, - templateValues: this.templates, - availableMlModels: availableMlModelUris, - id: callbackId, - timeoutSecs: qs.config.timeoutSecs, - }; - - const dataset: messages.Dataset = { - dbDir: dbItem.contents.datasetUri.fsPath, - workingSet: 'default' - }; - if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) { - await qs.sendRequest(messages.startLog, { - db: dataset, - logPath: this.evalLogPath, - }); - - } - const params: messages.EvaluateQueriesParams = { - db: dataset, - evaluateId: callbackId, - queries: [queryToRun], - stopOnError: false, - useSequenceHint: false - }; - try { - await qs.sendRequest(messages.runQueries, params, token, progress); - if (qs.config.customLogDirectory) { - void showAndLogWarningMessage( - `Custom log directories are no longer supported. The "codeQL.runningQueries.customLogDirectory" setting is deprecated. Unset the setting to stop seeing this message. Query logs saved to ${this.logPath}.` - ); - } - } finally { - qs.unRegisterCallback(callbackId); - if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) { - await qs.sendRequest(messages.endLog, { - db: dataset, - logPath: this.evalLogPath, - }); - if (await this.hasEvalLog()) { - queryInfo.evalLogLocation = this.evalLogPath; - queryInfo.evalLogSummaryLocation = await this.generateHumanReadableLogSummary(qs); - void this.logEndSummary(queryInfo.evalLogSummaryLocation, qs); // Logged asynchrnously - - if (config.isCanary()) { // Generate JSON summary for viewer. - await qs.cliServer.generateJsonLogSummary(this.evalLogPath, this.jsonEvalLogSummaryPath); - queryInfo.jsonEvalLogSummaryLocation = this.jsonEvalLogSummaryPath; - await generateSummarySymbolsFile(this.evalLogSummaryPath, this.evalLogSummarySymbolsPath); - queryInfo.evalLogSummarySymbolsLocation = this.evalLogSummarySymbolsPath; - } - } else { - void showAndLogWarningMessage(`Failed to write structured evaluator log to ${this.evalLogPath}.`); - } - } - } - return result || { - evaluationTime: 0, - message: 'No result from server', - queryId: -1, - runId: callbackId, - resultType: messages.QueryResultType.OTHER_ERROR - }; - } - - async compile( - qs: qsClient.QueryServerClient, - program: messages.QlProgram, - progress: ProgressCallback, - token: CancellationToken, - ): Promise { - let compiled: messages.CheckQueryResult | undefined; - try { - const target = this.quickEvalPosition ? { - quickEval: { quickEvalPos: this.quickEvalPosition } - } : { query: {} }; - const params: messages.CompileQueryParams = { - compilationOptions: { - computeNoLocationUrls: true, - failOnWarnings: false, - fastCompilation: false, - includeDilInQlo: true, - localChecking: false, - noComputeGetUrl: false, - noComputeToString: false, - computeDefaultStrings: true, - emitDebugInfo: true - }, - extraOptions: { - timeoutSecs: qs.config.timeoutSecs - }, - queryToCheck: program, - resultPath: this.compiledQueryPath, - target, - }; - - compiled = await qs.sendRequest(messages.compileQuery, params, token, progress); - } finally { - void qs.logger.log(' - - - COMPILATION DONE - - - ', { additionalLogLocation: this.logPath }); - } - return (compiled?.messages || []).filter(msg => msg.severity === messages.Severity.ERROR); - } - - /** - * Holds if this query can in principle produce interpreted results. - */ - canHaveInterpretedResults(): boolean { - if (!this.databaseHasMetadataFile) { - void logger.log('Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.'); - return false; - } - - const kind = this.metadata?.kind; - const hasKind = !!kind; - if (!hasKind) { - void logger.log('Cannot produce interpreted results since the query does not have @kind metadata.'); - return false; - } - - // Graph queries only return interpreted results if we are in canary mode. - if (kind === 'graph') { - return config.isCanary(); - } - - // table is the default query kind. It does not produce interpreted results. - // any query kind that is not table can, in principle, produce interpreted results. - return kind !== 'table'; - } - - /** - * Holds if this query actually has produced interpreted results. - */ - async hasInterpretedResults(): Promise { - return fs.pathExists(this.resultsPaths.interpretedResultsPath); - } - - /** - * Holds if this query already has DIL produced - */ - async hasDil(): Promise { - return fs.pathExists(this.dilPath); - } - - /** - * Holds if this query already has CSV results produced - */ - async hasCsv(): Promise { - return fs.pathExists(this.csvPath); - } - - /** - * Returns the path to the DIL file produced by this query. If the query has not yet produced DIL, - * this will return first create the DIL file and then return the path to the DIL file. - */ - async ensureDilPath(qs: qsClient.QueryServerClient): Promise { - if (await this.hasDil()) { - return this.dilPath; - } - - if (!(await fs.pathExists(this.compiledQueryPath))) { - throw new Error( - `Cannot create DIL because compiled query is missing. ${this.compiledQueryPath}` - ); - } - - await qs.cliServer.generateDil(this.compiledQueryPath, this.dilPath); - return this.dilPath; - } - - /** - * Holds if this query already has a completed structured evaluator log - */ - async hasEvalLog(): Promise { - return fs.pathExists(this.evalLogPath); - } - - /** - * Calls the appropriate CLI command to generate a human-readable log summary. - * @param qs The query server client. - * @returns The path to the log summary, or `undefined` if the summary could not be generated. - */ - private async generateHumanReadableLogSummary(qs: qsClient.QueryServerClient): Promise { - try { - await qs.cliServer.generateLogSummary(this.evalLogPath, this.evalLogSummaryPath, this.evalLogEndSummaryPath); - return this.evalLogSummaryPath; - - } catch (e) { - const err = asError(e); - void showAndLogWarningMessage(`Failed to generate human-readable structured evaluator log summary. Reason: ${err.message}`); - return undefined; - } - } - - /** - * Logs the end summary to the Output window and log file. - * @param logSummaryPath Path to the human-readable log summary - * @param qs The query server client. - */ - private async logEndSummary(logSummaryPath: string | undefined, qs: qsClient.QueryServerClient): Promise { - if (logSummaryPath === undefined) { - // Failed to generate the log, so we don't expect an end summary either. - return; - } - - try { - const endSummaryContent = await fs.readFile(this.evalLogEndSummaryPath, 'utf-8'); - void qs.logger.log(' --- Evaluator Log Summary --- ', { additionalLogLocation: this.logPath }); - void qs.logger.log(endSummaryContent, { additionalLogLocation: this.logPath }); - } catch (e) { - void showAndLogWarningMessage(`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`); - } - } - - /** - * Creates the CSV file containing the results of this query. This will only be called if the query - * does not have interpreted results and the CSV file does not already exist. - * - * @return Promise if the operation creates the file. Promise if the operation does - * not create the file. - * - * @throws Error if the operation fails. - */ - async exportCsvResults(qs: qsClient.QueryServerClient, csvPath: string): Promise { - const resultSet = await this.chooseResultSet(qs); - if (!resultSet) { - void showAndLogWarningMessage('Query has no result set.'); - return false; - } - let stopDecoding = false; - const out = fs.createWriteStream(csvPath); - - const promise: Promise = new Promise((resolve, reject) => { - out.on('finish', () => resolve(true)); - out.on('error', () => { - if (!stopDecoding) { - stopDecoding = true; - reject(new Error(`Failed to write CSV results to ${csvPath}`)); - } - }); - }); - - let nextOffset: number | undefined = 0; - do { - const chunk: DecodedBqrsChunk = await qs.cliServer.bqrsDecode(this.resultsPaths.resultsPath, resultSet, { - pageSize: 100, - offset: nextOffset, - }); - chunk.tuples.forEach((tuple) => { - out.write(tuple.map((v, i) => - chunk.columns[i].kind === 'String' - ? `"${typeof v === 'string' ? v.replaceAll('"', '""') : v}"` - : v - ).join(',') + '\n'); - }); - nextOffset = chunk.next; - } while (nextOffset && !stopDecoding); - out.end(); - - return promise; - } - - /** - * Choose the name of the result set to run. If the `#select` set exists, use that. Otherwise, - * arbitrarily choose the first set. Most of the time, this will be correct. - * - * If the query has no result sets, then return undefined. - */ - async chooseResultSet(qs: qsClient.QueryServerClient) { - const resultSets = (await qs.cliServer.bqrsInfo(this.resultsPaths.resultsPath, 0))['result-sets']; - if (!resultSets.length) { - return undefined; - } - if (resultSets.find(r => r.name === SELECT_QUERY_NAME)) { - return SELECT_QUERY_NAME; - } - return resultSets[0].name; - } - /** - * Returns the path to the CSV alerts interpretation of this query results. If CSV results have - * not yet been produced, this will return first create the CSV results and then return the path. - * - * This method only works for queries with interpreted results. - */ - async ensureCsvAlerts(qs: qsClient.QueryServerClient, dbm: DatabaseManager): Promise { - if (await this.hasCsv()) { - return this.csvPath; - } - - const dbItem = dbm.findDatabaseItem(Uri.file(this.dbItemPath)); - if (!dbItem) { - throw new Error(`Cannot produce CSV results because database is missing. ${this.dbItemPath}`); - } - - let sourceInfo; - if (dbItem.sourceArchive !== undefined) { - sourceInfo = { - sourceArchive: dbItem.sourceArchive.fsPath, - sourceLocationPrefix: await dbItem.getSourceLocationPrefix( - qs.cliServer - ), - }; - } - - await qs.cliServer.generateResultsCsv(ensureMetadataIsComplete(this.metadata), this.resultsPaths.resultsPath, this.csvPath, sourceInfo); - return this.csvPath; - } - - /** - * Cleans this query's results directory. - */ - async deleteQuery(): Promise { - await fs.remove(this.querySaveDir); - } -} - -export interface QueryWithResults { - readonly query: QueryEvaluationInfo; - readonly result: messages.EvaluationResult; - readonly logFileLocation?: string; - readonly dispose: () => void; -} - -export async function clearCacheInDatabase( - qs: qsClient.QueryServerClient, - dbItem: DatabaseItem, - progress: ProgressCallback, - token: CancellationToken, -): Promise { - if (dbItem.contents === undefined) { - throw new Error('Can\'t clear the cache in an invalid database.'); - } - - const db: messages.Dataset = { - dbDir: dbItem.contents.datasetUri.fsPath, - workingSet: 'default', - }; - - const params: messages.ClearCacheParams = { - dryRun: false, - db, - }; - - return qs.sendRequest(messages.clearCache, params, token, progress); -} - -/** - * @param filePath This needs to be equivalent to Java's `Path.toRealPath(NO_FOLLOW_LINKS)` - */ -async function convertToQlPath(filePath: string): Promise { - if (process.platform === 'win32') { - - if (path.parse(filePath).root === filePath) { - // Java assumes uppercase drive letters are canonical. - return filePath.toUpperCase(); - } else { - const dir = await convertToQlPath(path.dirname(filePath)); - const fileName = path.basename(filePath); - const fileNames = await fs.readdir(dir); - for (const name of fileNames) { - // Leave the locale argument empty so that the default OS locale is used. - // We do this because this operation works on filesystem entities, which - // use the os locale, regardless of the locale of the running VS Code instance. - if (fileName.localeCompare(name, undefined, { sensitivity: 'accent' }) === 0) { - return path.join(dir, name); - } - } - } - throw new Error('Can\'t convert path to form suitable for QL:' + filePath); - } else { - return filePath; - } -} - - - -/** Gets the selected position within the given editor. */ -async function getSelectedPosition(editor: TextEditor, range?: Range): Promise { - const selectedRange = range || editor.selection; - const pos = selectedRange.start; - const posEnd = selectedRange.end; - // Convert from 0-based to 1-based line and column numbers. - return { - fileName: await convertToQlPath(editor.document.fileName), - line: pos.line + 1, - column: pos.character + 1, - endLine: posEnd.line + 1, - endColumn: posEnd.character + 1 - }; -} - -/** - * Compare the dbscheme implied by the query `query` and that of the current database. - * - If they are compatible, do nothing. - * - If they are incompatible but the database can be upgraded, suggest that upgrade. - * - If they are incompatible and the database cannot be upgraded, throw an error. - */ -async function checkDbschemeCompatibility( - cliServer: cli.CodeQLCliServer, - qs: qsClient.QueryServerClient, - query: QueryEvaluationInfo, - qlProgram: messages.QlProgram, - dbItem: DatabaseItem, - progress: ProgressCallback, - token: CancellationToken, -): Promise { - const searchPath = getOnDiskWorkspaceFolders(); - - if (dbItem.contents?.dbSchemeUri !== undefined) { - const { finalDbscheme } = await cliServer.resolveUpgrades(dbItem.contents.dbSchemeUri.fsPath, searchPath, false); - const hash = async function(filename: string): Promise { - return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex'); - }; - - // At this point, we have learned about three dbschemes: - - // the dbscheme of the actual database we're querying. - const dbschemeOfDb = await hash(dbItem.contents.dbSchemeUri.fsPath); - - // the dbscheme of the query we're running, including the library we've resolved it to use. - const dbschemeOfLib = await hash(query.queryDbscheme); - - // the database we're able to upgrade to - const upgradableTo = await hash(finalDbscheme); - - if (upgradableTo != dbschemeOfLib) { - reportNoUpgradePath(qlProgram, query); - } - - if (upgradableTo == dbschemeOfLib && - dbschemeOfDb != dbschemeOfLib) { - // Try to upgrade the database - await upgradeDatabaseExplicit( - qs, - dbItem, - progress, - token - ); - } - } -} - -function reportNoUpgradePath(qlProgram: messages.QlProgram, query: QueryEvaluationInfo): void { - throw new Error( - `Query ${qlProgram.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.` - ); -} - -/** - * Compile a non-destructive upgrade. - */ -async function compileNonDestructiveUpgrade( - qs: qsClient.QueryServerClient, - upgradeTemp: tmp.DirectoryResult, - query: QueryEvaluationInfo, - qlProgram: messages.QlProgram, - dbItem: DatabaseItem, - progress: ProgressCallback, - token: CancellationToken, -): Promise { - - if (!dbItem?.contents?.dbSchemeUri) { - throw new Error('Database is invalid, and cannot be upgraded.'); - } - - // When packaging is used, dependencies may exist outside of the workspace and they are always on the resolved search path. - // When packaging is not used, all dependencies are in the workspace. - const upgradesPath = (await qs.cliServer.cliConstraints.supportsPackaging()) - ? qlProgram.libraryPath - : getOnDiskWorkspaceFolders(); - - const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades( - dbItem.contents.dbSchemeUri.fsPath, - upgradesPath, - true, - query.queryDbscheme - ); - - if (!matchesTarget) { - reportNoUpgradePath(qlProgram, query); - } - const result = await compileDatabaseUpgradeSequence(qs, dbItem, scripts, upgradeTemp, progress, token); - if (result.compiledUpgrade === undefined) { - const error = result.error || '[no error message available]'; - throw new Error(error); - } - // We can upgrade to the actual target - qlProgram.dbschemePath = query.queryDbscheme; - // We are new enough that we will always support single file upgrades. - return result.compiledUpgrade; -} - -/** - * Prompts the user to save `document` if it has unsaved changes. - * - * @param document The document to save. - * - * @returns true if we should save changes and false if we should continue without saving changes. - * @throws UserCancellationException if we should abort whatever operation triggered this prompt - */ -async function promptUserToSaveChanges(document: TextDocument): Promise { - if (document.isDirty) { - if (config.AUTOSAVE_SETTING.getValue()) { - return true; - } - else { - const yesItem = { title: 'Yes', isCloseAffordance: false }; - const alwaysItem = { title: 'Always Save', isCloseAffordance: false }; - const noItem = { title: 'No (run version on disk)', isCloseAffordance: false }; - const cancelItem = { title: 'Cancel', isCloseAffordance: true }; - const message = 'Query file has unsaved changes. Save now?'; - const chosenItem = await window.showInformationMessage( - message, - { modal: true }, - yesItem, alwaysItem, noItem, cancelItem - ); - - if (chosenItem === alwaysItem) { - await config.AUTOSAVE_SETTING.updateValue(true, ConfigurationTarget.Workspace); - return true; - } - - if (chosenItem === yesItem) { - return true; - } - - if (chosenItem === cancelItem) { - throw new UserCancellationException('Query run cancelled.', true); - } - } - } - return false; -} - -/** - * Determines which QL file to run during an invocation of `Run Query` or `Quick Evaluation`, as follows: - * - If the command was called by clicking on a file, then use that file. - * - Otherwise, use the file open in the current editor. - * - In either case, prompt the user to save the file if it is open with unsaved changes. - * - For `Quick Evaluation`, ensure the selected file is also the one open in the editor, - * and use the selected region. - * @param selectedResourceUri The selected resource when the command was run. - * @param quickEval Whether the command being run is `Quick Evaluation`. -*/ -export async function determineSelectedQuery( - selectedResourceUri: Uri | undefined, - quickEval: boolean, - range?: Range -): Promise { - const editor = window.activeTextEditor; - - // Choose which QL file to use. - let queryUri: Uri; - if (selectedResourceUri) { - // A resource was passed to the command handler, so use it. - queryUri = selectedResourceUri; - } else { - // No resource was passed to the command handler, so obtain it from the active editor. - // This usually happens when the command is called from the Command Palette. - if (editor === undefined) { - throw new Error('No query was selected. Please select a query and try again.'); - } else { - queryUri = editor.document.uri; - } - } - - if (queryUri.scheme !== 'file') { - throw new Error('Can only run queries that are on disk.'); - } - const queryPath = queryUri.fsPath; - - if (quickEval) { - if (!(queryPath.endsWith('.ql') || queryPath.endsWith('.qll'))) { - throw new Error('The selected resource is not a CodeQL file; It should have the extension ".ql" or ".qll".'); - } - } else { - if (!(queryPath.endsWith('.ql'))) { - throw new Error('The selected resource is not a CodeQL query file; It should have the extension ".ql".'); - } - } - - // Whether we chose the file from the active editor or from a context menu, - // if the same file is open with unsaved changes in the active editor, - // then prompt the user to save it first. - if (editor !== undefined && editor.document.uri.fsPath === queryPath) { - if (await promptUserToSaveChanges(editor.document)) { - await editor.document.save(); - } - } - - let quickEvalPosition: messages.Position | undefined = undefined; - let quickEvalText: string | undefined = undefined; - if (quickEval) { - if (editor == undefined) { - throw new Error('Can\'t run quick evaluation without an active editor.'); - } - if (editor.document.fileName !== queryPath) { - // For Quick Evaluation we expect these to be the same. - // Report an error if we end up in this (hopefully unlikely) situation. - throw new Error('The selected resource for quick evaluation should match the active editor.'); - } - quickEvalPosition = await getSelectedPosition(editor, range); - if (!editor.selection?.isEmpty) { - quickEvalText = editor.document.getText(editor.selection); - } else { - // capture the entire line if the user didn't select anything - const line = editor.document.lineAt(editor.selection.active.line); - quickEvalText = line.text.trim(); - } - } - - return { queryPath, quickEvalPosition, quickEvalText }; -} - -export async function compileAndRunQueryAgainstDatabase( - cliServer: cli.CodeQLCliServer, - qs: qsClient.QueryServerClient, - dbItem: DatabaseItem, - initialInfo: InitialQueryInfo, - queryStorageDir: string, - progress: ProgressCallback, - token: CancellationToken, - templates?: messages.TemplateDefinitions, - queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query. -): Promise { - if (!dbItem.contents || !dbItem.contents.dbSchemeUri) { - throw new Error(`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`); - } - - // Get the workspace folder paths. - const diskWorkspaceFolders = getOnDiskWorkspaceFolders(); - // Figure out the library path for the query. - const packConfig = await cliServer.resolveLibraryPath(diskWorkspaceFolders, initialInfo.queryPath); - - if (!packConfig.dbscheme) { - throw new Error('Could not find a database scheme for this query. Please check that you have a valid qlpack.yml file for this query, which refers to a database scheme either in the `dbscheme` field or through one of its dependencies.'); - } - - // Check whether the query has an entirely different schema from the - // database. (Queries that merely need the database to be upgraded - // won't trigger this check) - // This test will produce confusing results if we ever change the name of the database schema files. - const querySchemaName = path.basename(packConfig.dbscheme); - const dbSchemaName = path.basename(dbItem.contents.dbSchemeUri.fsPath); - if (querySchemaName != dbSchemaName) { - void logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`); - throw new Error(`The query ${path.basename(initialInfo.queryPath)} cannot be run against the selected database (${dbItem.name}): their target languages are different. Please select a different database and try again.`); - } - - const qlProgram: messages.QlProgram = { - // The project of the current document determines which library path - // we use. The `libraryPath` field in this server message is relative - // to the workspace root, not to the project root. - libraryPath: packConfig.libraryPath, - // Since we are compiling and running a query against a database, - // we use the database's DB scheme here instead of the DB scheme - // from the current document's project. - dbschemePath: dbItem.contents.dbSchemeUri.fsPath, - queryPath: initialInfo.queryPath - }; - - // Read the query metadata if possible, to use in the UI. - const metadata = await tryGetQueryMetadata(cliServer, qlProgram.queryPath); - - let availableMlModels: cli.MlModelInfo[] = []; - if (!await cliServer.cliConstraints.supportsResolveMlModels()) { - void logger.log('Resolving ML models is unsupported by this version of the CLI. Running the query without any ML models.'); - } else { - try { - availableMlModels = (await cliServer.resolveMlModels(diskWorkspaceFolders, initialInfo.queryPath)).models; - if (availableMlModels.length) { - void logger.log(`Found available ML models at the following paths: ${availableMlModels.map(x => `'${x.path}'`).join(', ')}.`); - } else { - void logger.log('Did not find any available ML models.'); - } - } catch (e) { - const message = `Couldn't resolve available ML models for ${qlProgram.queryPath}. Running the ` + - `query without any ML models: ${e}.`; - void showAndLogErrorMessage(message); - } - } - - const hasMetadataFile = (await dbItem.hasMetadataFile()); - const query = new QueryEvaluationInfo( - path.join(queryStorageDir, initialInfo.id), - dbItem.databaseUri.fsPath, - hasMetadataFile, - packConfig.dbscheme, - initialInfo.quickEvalPosition, - metadata, - templates - ); - await query.createTimestampFile(); - - let upgradeDir: tmp.DirectoryResult | undefined; - try { - let upgradeQlo; - if (await qs.cliServer.cliConstraints.supportsNonDestructiveUpgrades()) { - upgradeDir = await tmp.dir({ dir: upgradesTmpDir, unsafeCleanup: true }); - upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, qlProgram, dbItem, progress, token); - } else { - await checkDbschemeCompatibility(cliServer, qs, query, qlProgram, dbItem, progress, token); - } - let errors; - try { - errors = await query.compile(qs, qlProgram, progress, token); - } catch (e) { - if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) { - return createSyntheticResult(query, 'Query cancelled', messages.QueryResultType.CANCELLATION); - } else { - throw e; - } - } - - if (errors.length === 0) { - const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token, queryInfo); - if (result.resultType !== messages.QueryResultType.SUCCESS) { - const message = result.message || 'Failed to run query'; - void logger.log(message); - void showAndLogErrorMessage(message); - } - return { - query, - result, - logFileLocation: result.logFileLocation, - dispose: () => { - qs.logger.removeAdditionalLogLocation(result.logFileLocation); - } - }; - } else { - // Error dialogs are limited in size and scrollability, - // so we include a general description of the problem, - // and direct the user to the output window for the detailed compilation messages. - // However we don't show quick eval errors there so we need to display them anyway. - void qs.logger.log( - `Failed to compile query ${initialInfo.queryPath} against database scheme ${qlProgram.dbschemePath}:`, - { additionalLogLocation: query.logPath } - ); - - const formattedMessages: string[] = []; - - for (const error of errors) { - const message = error.message || '[no error message available]'; - const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`; - formattedMessages.push(formatted); - void qs.logger.log(formatted, { additionalLogLocation: query.logPath }); - } - if (initialInfo.isQuickEval && formattedMessages.length <= 2) { - // If there are more than 2 error messages, they will not be displayed well in a popup - // and will be trimmed by the function displaying the error popup. Accordingly, we only - // try to show the errors if there are 2 or less, otherwise we direct the user to the log. - void showAndLogErrorMessage('Quick evaluation compilation failed: ' + formattedMessages.join('\n')); - } else { - void showAndLogErrorMessage((initialInfo.isQuickEval ? 'Quick evaluation' : 'Query') + compilationFailedErrorTail); - } - - return createSyntheticResult(query, 'Query had compilation errors', messages.QueryResultType.OTHER_ERROR); - } - } finally { - try { - await upgradeDir?.cleanup(); - } catch (e) { - void qs.logger.log( - `Could not clean up the upgrades dir. Reason: ${getErrorMessage(e)}`, - { additionalLogLocation: query.logPath } - ); - } - } -} - -/** - * Determines the initial information for a query. This is everything of interest - * we know about this query that is available before it is run. - * - * @param selectedQueryUri The Uri of the document containing the query to be run. - * @param databaseInfo The database to run the query against. - * @param isQuickEval true if this is a quick evaluation. - * @param range the selection range of the query to be run. Only used if isQuickEval is true. - * @returns The initial information for the query to be run. - */ -export async function createInitialQueryInfo( - selectedQueryUri: Uri | undefined, - databaseInfo: DatabaseInfo, - isQuickEval: boolean, - range?: Range -): Promise { - // Determine which query to run, based on the selection and the active editor. - const { queryPath, quickEvalPosition, quickEvalText } = await determineSelectedQuery(selectedQueryUri, isQuickEval, range); - - return { - queryPath, - isQuickEval, - isQuickQuery: isQuickQueryPath(queryPath), - databaseInfo, - id: `${path.basename(queryPath)}-${nanoid()}`, - start: new Date(), - ... (isQuickEval ? { - queryText: quickEvalText!, // if this query is quick eval, it must have quick eval text - quickEvalPosition: quickEvalPosition - } : { - queryText: await fs.readFile(queryPath, 'utf8') - }) - }; -} - - -const compilationFailedErrorTail = ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' + - ' and the query and database use the same target language. For more details on the error, go to View > Output,' + - ' and choose CodeQL Query Server from the dropdown.'; - -/** - * Create a synthetic result for a query that failed to compile. - */ -function createSyntheticResult( - query: QueryEvaluationInfo, - message: string, - resultType: number -): QueryWithResults { - return { - query, - result: { - evaluationTime: 0, - resultType: resultType, - queryId: -1, - runId: -1, - message - }, - dispose: () => { /**/ }, - }; -} diff --git a/extensions/ql-vscode/src/vscode-tests/cli-integration/query.test.ts b/extensions/ql-vscode/src/vscode-tests/cli-integration/legacy-query.test.ts similarity index 87% rename from extensions/ql-vscode/src/vscode-tests/cli-integration/query.test.ts rename to extensions/ql-vscode/src/vscode-tests/cli-integration/legacy-query.test.ts index bf1fdcdf8..d94800aa5 100644 --- a/extensions/ql-vscode/src/vscode-tests/cli-integration/query.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/cli-integration/legacy-query.test.ts @@ -4,14 +4,16 @@ import * as path from 'path'; import * as tmp from 'tmp'; import * as url from 'url'; import { CancellationTokenSource } from 'vscode-jsonrpc'; -import * as messages from '../../pure/messages'; -import * as qsClient from '../../queryserver-client'; +import * as messages from '../../pure/legacy-messages'; +import * as qsClient from '../../legacy-query-server/queryserver-client'; import * as cli from '../../cli'; import { CellValue } from '../../pure/bqrs-cli-types'; import { extensions } from 'vscode'; import { CodeQLExtensionInterface } from '../../extension'; import { fail } from 'assert'; import { skipIfNoCodeQL } from '../ensureCli'; +import { QueryServerClient } from '../../legacy-query-server/queryserver-client'; +import { logger, ProgressReporter } from '../../logging'; const baseDir = path.join(__dirname, '../../../test/data'); @@ -88,7 +90,7 @@ const db: messages.Dataset = { workingSet: 'default', }; -describe('using the query server', function() { +describe('using the legacy query server', function() { before(function() { skipIfNoCodeQL(this); }); @@ -97,28 +99,37 @@ describe('using the query server', function() { // ensure they are all written with standard anonymous functions. this.timeout(20000); + const nullProgressReporter: ProgressReporter = { report: () => { /** ignore */ } }; + let qs: qsClient.QueryServerClient; let cliServer: cli.CodeQLCliServer; - const queryServerStarted = new Checkpoint(); - beforeEach(async () => { + before(async () => { try { const extension = await extensions.getExtension>('GitHub.vscode-codeql')!.activate(); - if ('cliServer' in extension && 'qs' in extension) { + if ('cliServer' in extension) { cliServer = extension.cliServer; - qs = extension.qs; cliServer.quiet = true; + + qs = new QueryServerClient({ + codeQlPath: (await extension.distributionManager.getCodeQlPathWithoutVersionCheck()) || '', + debug: false, + cacheSize: 0, + numThreads: 1, + saveCache: false, + timeoutSecs: 0 + }, cliServer, { + contextStoragePath: tmpDir.name, + logger + }, task => task(nullProgressReporter, new CancellationTokenSource().token)); + await qs.startQueryServer(); } else { throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.'); } } catch (e) { fail(e as Error); } - }); - it('should be able to start the query server', async function() { - await qs.startQueryServer(); - await queryServerStarted.resolve(); }); for (const queryTestCase of queryTestCases) { @@ -134,7 +145,6 @@ describe('using the query server', function() { }); it(`should be able to compile query ${queryName}`, async function() { - await queryServerStarted.done(); expect(fs.existsSync(queryTestCase.queryPath)).to.be.true; try { const qlProgram: messages.QlProgram = { diff --git a/extensions/ql-vscode/src/vscode-tests/cli-integration/queries.test.ts b/extensions/ql-vscode/src/vscode-tests/cli-integration/queries.test.ts index fe4e266ce..dc7b17d27 100644 --- a/extensions/ql-vscode/src/vscode-tests/cli-integration/queries.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/cli-integration/queries.test.ts @@ -10,12 +10,11 @@ import { DatabaseItem, DatabaseManager } from '../../databases'; import { CodeQLExtensionInterface } from '../../extension'; import { cleanDatabases, dbLoc, storagePath } from './global.helper'; import { importArchiveDatabase } from '../../databaseFetcher'; -import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo } from '../../run-queries'; import { CodeQLCliServer } from '../../cli'; -import { QueryServerClient } from '../../queryserver-client'; import { skipIfNoCodeQL } from '../ensureCli'; -import { QueryResultType } from '../../pure/messages'; import { tmpDir } from '../../helpers'; +import { createInitialQueryInfo } from '../../run-queries-shared'; +import { QueryRunner } from '../../queryRunner'; /** @@ -31,7 +30,7 @@ describe('Queries', function() { let dbItem: DatabaseItem; let databaseManager: DatabaseManager; let cli: CodeQLCliServer; - let qs: QueryServerClient; + let qs: QueryRunner; let sandbox: sinon.SinonSandbox; let progress: sinon.SinonSpy; let token: CancellationToken; @@ -104,9 +103,7 @@ describe('Queries', function() { it('should run a query', async () => { try { const queryPath = path.join(__dirname, 'data', 'simple-query.ql'); - const result = await compileAndRunQueryAgainstDatabase( - cli, - qs, + const result = qs.compileAndRunQueryAgainstDatabase( dbItem, await mockInitialQueryInfo(queryPath), path.join(tmpDir.name, 'mock-storage-path'), @@ -115,7 +112,7 @@ describe('Queries', function() { ); // just check that the query was successful - expect(result.result.resultType).to.eq(QueryResultType.SUCCESS); + expect((await result).sucessful).to.eq(true); } catch (e) { console.error('Test Failed'); fail(e as Error); @@ -127,9 +124,7 @@ describe('Queries', function() { try { await commands.executeCommand('codeQL.restartQueryServer'); const queryPath = path.join(__dirname, 'data', 'simple-query.ql'); - const result = await compileAndRunQueryAgainstDatabase( - cli, - qs, + const result = await qs.compileAndRunQueryAgainstDatabase( dbItem, await mockInitialQueryInfo(queryPath), path.join(tmpDir.name, 'mock-storage-path'), @@ -137,9 +132,7 @@ describe('Queries', function() { token ); - // this message would indicate that the databases were not properly reregistered - expect(result.result.message).not.to.eq('No result from server'); - expect(result.result.resultType).to.eq(QueryResultType.SUCCESS); + expect(result.sucessful).to.eq(true); } catch (e) { console.error('Test Failed'); fail(e as Error); diff --git a/extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts b/extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts index 0c79a3b6b..161426f20 100644 --- a/extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts @@ -14,12 +14,11 @@ import { findSourceArchive } from '../../databases'; import { Logger } from '../../logging'; -import { QueryServerClient } from '../../queryserver-client'; -import { registerDatabases } from '../../pure/messages'; import { ProgressCallback } from '../../commandRunner'; import { CodeQLCliServer } from '../../cli'; import { encodeArchiveBasePath, encodeSourceArchiveUri } from '../../archive-filesystem-provider'; import { testDisposeHandler } from '../test-dispose-handler'; +import { QueryRunner } from '../../queryRunner'; describe('databases', () => { @@ -33,7 +32,8 @@ describe('databases', () => { let updateSpy: sinon.SinonSpy; let getSpy: sinon.SinonStub; let dbChangedHandler: sinon.SinonSpy; - let sendRequestSpy: sinon.SinonSpy; + let registerSpy: sinon.SinonSpy; + let deregisterSpy: sinon.SinonSpy; let supportsDatabaseRegistrationSpy: sinon.SinonStub; let supportsLanguageNameSpy: sinon.SinonStub; let resolveDatabaseSpy: sinon.SinonStub; @@ -48,7 +48,8 @@ describe('databases', () => { updateSpy = sandbox.spy(); getSpy = sandbox.stub(); getSpy.returns([]); - sendRequestSpy = sandbox.stub(); + registerSpy = sandbox.stub(); + deregisterSpy = sandbox.stub(); dbChangedHandler = sandbox.spy(); supportsDatabaseRegistrationSpy = sandbox.stub(); supportsDatabaseRegistrationSpy.resolves(true); @@ -65,9 +66,10 @@ describe('databases', () => { storagePath: dir.name } as unknown as ExtensionContext, { - sendRequest: sendRequestSpy, - onDidStartQueryServer: () => { /**/ } - } as unknown as QueryServerClient, + registerDatabase: registerSpy, + deregisterDatabase: deregisterSpy, + onStart: () => { /**/ } + } as unknown as QueryRunner, { cliConstraints: { supportsLanguageName: supportsLanguageNameSpy, @@ -259,12 +261,6 @@ describe('databases', () => { // similar test as above, but also check the call to sendRequestSpy to make sure they send the // registration messages. const mockDbItem = createMockDB(); - const registration = { - databases: [{ - dbDir: mockDbItem.contents!.datasetUri.fsPath, - workingSet: 'default' - }] - }; sandbox.stub(fs, 'remove').resolves(); @@ -274,7 +270,7 @@ describe('databases', () => { mockDbItem ); // Should have registered this database - expect(sendRequestSpy).to.have.been.calledWith(registerDatabases, registration, {}, {}); + expect(registerSpy).to.have.been.calledWith({}, {}, mockDbItem); await databaseManager.removeDatabaseItem( {} as ProgressCallback, @@ -283,31 +279,7 @@ describe('databases', () => { ); // Should have deregistered this database - expect(sendRequestSpy).to.have.been.calledWith(registerDatabases, registration, {}, {}); - }); - - it('should avoid registration when query server does not support it', async () => { - // similar test as above, but now pretend query server doesn't support registration - supportsDatabaseRegistrationSpy.resolves(false); - const mockDbItem = createMockDB(); - sandbox.stub(fs, 'remove').resolves(); - - await (databaseManager as any).addDatabaseItem( - {} as ProgressCallback, - {} as CancellationToken, - mockDbItem - ); - // Should NOT have registered this database - expect(sendRequestSpy).not.to.have.been.called; - - await databaseManager.removeDatabaseItem( - {} as ProgressCallback, - {} as CancellationToken, - mockDbItem - ); - - // Should NOT have deregistered this database - expect(sendRequestSpy).not.to.have.been.called; + expect(deregisterSpy).to.have.been.calledWith({}, {}, mockDbItem); }); }); diff --git a/extensions/ql-vscode/src/vscode-tests/minimal-workspace/determining-selected-query-test.ts b/extensions/ql-vscode/src/vscode-tests/minimal-workspace/determining-selected-query-test.ts index 9c56bed79..888108a7d 100644 --- a/extensions/ql-vscode/src/vscode-tests/minimal-workspace/determining-selected-query-test.ts +++ b/extensions/ql-vscode/src/vscode-tests/minimal-workspace/determining-selected-query-test.ts @@ -2,7 +2,7 @@ import { expect } from 'chai'; import * as path from 'path'; import * as vscode from 'vscode'; import { Uri } from 'vscode'; -import { determineSelectedQuery } from '../../run-queries'; +import { determineSelectedQuery } from '../../run-queries-shared'; async function showQlDocument(name: string): Promise { const folderPath = vscode.workspace.workspaceFolders![0].uri.fsPath; diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/astBuilder.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/astBuilder.test.ts index f5e178ff0..c1a7074bc 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/astBuilder.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/astBuilder.test.ts @@ -3,10 +3,10 @@ import { expect } from 'chai'; import * as sinon from 'sinon'; import AstBuilder from '../../../contextual/astBuilder'; -import { QueryWithResults } from '../../../run-queries'; import { CodeQLCliServer } from '../../../cli'; import { DatabaseItem } from '../../../databases'; import { Uri } from 'vscode'; +import { QueryWithResults } from '../../../run-queries-shared'; /** * diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts index eb4f61b37..2f9a6ba42 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts @@ -7,10 +7,8 @@ import * as sinon from 'sinon'; import { logger } from '../../logging'; import { registerQueryHistoryScubber } from '../../query-history-scrubber'; import { QueryHistoryManager, HistoryTreeDataProvider, SortOrder } from '../../query-history'; -import { QueryEvaluationInfo, QueryWithResults } from '../../run-queries'; +import { QueryEvaluationInfo, QueryWithResults } from '../../run-queries-shared'; import { QueryHistoryConfig, QueryHistoryConfigListener } from '../../config'; -import * as messages from '../../pure/messages'; -import { QueryServerClient } from '../../queryserver-client'; import { LocalQueryInfo, InitialQueryInfo } from '../../query-results'; import { DatabaseManager } from '../../databases'; import * as tmp from 'tmp-promise'; @@ -21,6 +19,7 @@ import { HistoryItemLabelProvider } from '../../history-item-label-provider'; import { RemoteQueriesManager } from '../../remote-queries/remote-queries-manager'; import { InterfaceManager } from '../../interface'; import { EvalLogViewer } from '../../eval-log-viewer'; +import { QueryRunner } from '../../queryRunner'; describe('query-history', () => { const mockExtensionLocation = path.join(tmpDir.name, 'mock-extension-location'); @@ -785,18 +784,15 @@ describe('query-history', () => { hasInterpretedResults: () => Promise.resolve(hasInterpretedResults), deleteQuery: sandbox.stub(), } as unknown as QueryEvaluationInfo, - result: { - resultType: didRunSuccessfully - ? messages.QueryResultType.SUCCESS - : messages.QueryResultType.OTHER_ERROR - } as messages.EvaluationResult, + sucessful: didRunSuccessfully, + message: 'foo', dispose: sandbox.spy() }; } async function createMockQueryHistory(allHistory: LocalQueryInfo[]) { const qhm = new QueryHistoryManager( - {} as QueryServerClient, + {} as QueryRunner, {} as DatabaseManager, localQueriesInterfaceManagerStub, remoteQueriesManagerStub, diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/query-results.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/query-results.test.ts index 89a23028e..3a6478d66 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/query-results.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/query-results.test.ts @@ -3,13 +3,14 @@ import * as path from 'path'; import * as fs from 'fs-extra'; import * as sinon from 'sinon'; import { LocalQueryInfo, InitialQueryInfo, interpretResultsSarif } from '../../query-results'; -import { QueryEvaluationInfo, QueryWithResults } from '../../run-queries'; -import { EvaluationResult, QueryResultType } from '../../pure/messages'; +import { QueryWithResults } from '../../run-queries-shared'; import { DatabaseInfo, SortDirection, SortedResultSetInfo } from '../../pure/interface-types'; import { CodeQLCliServer, SourceInfo } from '../../cli'; import { CancellationTokenSource, Uri } from 'vscode'; import { tmpDir } from '../../helpers'; import { slurpQueryHistory, splatQueryHistory } from '../../query-serialization'; +import { formatLegacyMessage, QueryInProgress } from '../../legacy-query-server/run-queries'; +import { EvaluationResult, QueryResultType } from '../../pure/legacy-messages'; describe('query-results', () => { let disposeSpy: sinon.SinonSpy; @@ -87,30 +88,34 @@ describe('query-results', () => { expect(completedQuery.getResultsPath('zxa')).to.eq('bxa'); }); - it('should get the statusString', () => { - const fqi = createMockFullQueryInfo('a', createMockQueryWithResults(queryPath, false)); - const completedQuery = fqi.completedQuery!; + it('should format the statusString', () => { - completedQuery.result.message = 'Tremendously'; - expect(completedQuery.statusString).to.eq('failed: Tremendously'); + const evalResult: EvaluationResult = { + resultType: QueryResultType.OTHER_ERROR, + evaluationTime: 12340, + queryId: 3, + runId: 1, + }; - completedQuery.result.resultType = QueryResultType.OTHER_ERROR; - expect(completedQuery.statusString).to.eq('failed: Tremendously'); + evalResult.message = 'Tremendously'; + expect(formatLegacyMessage(evalResult)).to.eq('failed: Tremendously'); - completedQuery.result.resultType = QueryResultType.CANCELLATION; - completedQuery.result.evaluationTime = 2345; - expect(completedQuery.statusString).to.eq('cancelled after 2 seconds'); + evalResult.resultType = QueryResultType.OTHER_ERROR; + expect(formatLegacyMessage(evalResult)).to.eq('failed: Tremendously'); - completedQuery.result.resultType = QueryResultType.OOM; - expect(completedQuery.statusString).to.eq('out of memory'); + evalResult.resultType = QueryResultType.CANCELLATION; + evalResult.evaluationTime = 2345; + expect(formatLegacyMessage(evalResult)).to.eq('cancelled after 2 seconds'); - completedQuery.result.resultType = QueryResultType.SUCCESS; - expect(completedQuery.statusString).to.eq('finished in 2 seconds'); + evalResult.resultType = QueryResultType.OOM; + expect(formatLegacyMessage(evalResult)).to.eq('out of memory'); - completedQuery.result.resultType = QueryResultType.TIMEOUT; - expect(completedQuery.statusString).to.eq('timed out after 2 seconds'); + evalResult.resultType = QueryResultType.SUCCESS; + expect(formatLegacyMessage(evalResult)).to.eq('finished in 2 seconds'); + + evalResult.resultType = QueryResultType.TIMEOUT; + expect(formatLegacyMessage(evalResult)).to.eq('timed out after 2 seconds'); }); - it('should updateSortState', async () => { // setup const fqi = createMockFullQueryInfo('a', createMockQueryWithResults(queryPath)); @@ -267,9 +272,6 @@ describe('query-results', () => { if (!('quickEvalPosition' in query)) { (query as any).quickEvalPosition = undefined; } - if (!('templates' in query)) { - (query as any).templates = undefined; - } } }); expectedHistory.forEach(info => { @@ -310,7 +312,7 @@ describe('query-results', () => { fs.mkdirpSync(queryPath); fs.writeFileSync(resultsPath, '', 'utf8'); - const query = new QueryEvaluationInfo( + const query = new QueryInProgress( queryPath, Uri.file(dbPath).fsPath, true, @@ -321,14 +323,10 @@ describe('query-results', () => { }, ); - const result = { - query, - result: { - evaluationTime: 12340, - resultType: didRunSuccessfully - ? QueryResultType.SUCCESS - : QueryResultType.OTHER_ERROR - } as EvaluationResult, + const result: QueryWithResults = { + query: query.queryEvalInfo, + sucessful: didRunSuccessfully, + message: 'foo', dispose: disposeSpy, }; diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/remote-queries/remote-query-history.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/remote-queries/remote-query-history.test.ts index 500088471..aac0d2070 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/remote-queries/remote-query-history.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/remote-queries/remote-query-history.test.ts @@ -8,7 +8,6 @@ import { QueryHistoryConfig } from '../../../config'; import { DatabaseManager } from '../../../databases'; import { tmpDir } from '../../../helpers'; import { QueryHistoryManager } from '../../../query-history'; -import { QueryServerClient } from '../../../queryserver-client'; import { Credentials } from '../../../authentication'; import { AnalysesResultsManager } from '../../../remote-queries/analyses-results-manager'; import { RemoteQueryResult } from '../../../remote-queries/shared/remote-query-result'; @@ -20,6 +19,7 @@ import { HistoryItemLabelProvider } from '../../../history-item-label-provider'; import { RemoteQueriesManager } from '../../../remote-queries/remote-queries-manager'; import { InterfaceManager } from '../../../interface'; import { EvalLogViewer } from '../../../eval-log-viewer'; +import { QueryRunner } from '../../../queryRunner'; /** * Tests for remote queries and how they interact with the query history manager. @@ -90,7 +90,7 @@ describe('Remote queries and query history manager', function() { remoteQueryResult1 = fs.readJSONSync(path.join(STORAGE_DIR, 'queries', rawQueryHistory[1].queryId, 'query-result.json')); qhm = new QueryHistoryManager( - {} as QueryServerClient, + {} as QueryRunner, {} as DatabaseManager, localQueriesInterfaceManagerStub, remoteQueriesManagerStub, diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/run-queries.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/run-queries.test.ts index 910b4c9d8..4547f9d74 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/run-queries.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/run-queries.test.ts @@ -4,13 +4,15 @@ import * as fs from 'fs-extra'; import * as sinon from 'sinon'; import { Uri } from 'vscode'; -import { QueryEvaluationInfo } from '../../run-queries'; -import { Severity, compileQuery } from '../../pure/messages'; +import { Severity, compileQuery, registerDatabases, deregisterDatabases } from '../../pure/legacy-messages'; import * as config from '../../config'; import { tmpDir } from '../../helpers'; -import { QueryServerClient } from '../../queryserver-client'; +import { QueryServerClient } from '../../legacy-query-server/queryserver-client'; import { CodeQLCliServer } from '../../cli'; import { SELECT_QUERY_NAME } from '../../contextual/locationFinder'; +import { QueryInProgress } from '../../legacy-query-server/run-queries'; +import { LegacyQueryRunner } from '../../legacy-query-server/legacyRunner'; +import { DatabaseItem } from '../../databases'; describe('run-queries', () => { let sandbox: sinon.SinonSandbox; @@ -29,55 +31,53 @@ describe('run-queries', () => { const info = createMockQueryInfo(true, saveDir); expect(info.compiledQueryPath).to.eq(path.join(saveDir, 'compiledQuery.qlo')); - expect(info.dilPath).to.eq(path.join(saveDir, 'results.dil')); - expect(info.resultsPaths.resultsPath).to.eq(path.join(saveDir, 'results.bqrs')); - expect(info.resultsPaths.interpretedResultsPath).to.eq(path.join(saveDir, 'interpretedResults.sarif')); + expect(info.queryEvalInfo.dilPath).to.eq(path.join(saveDir, 'results.dil')); + expect(info.queryEvalInfo.resultsPaths.resultsPath).to.eq(path.join(saveDir, 'results.bqrs')); + expect(info.queryEvalInfo.resultsPaths.interpretedResultsPath).to.eq(path.join(saveDir, 'interpretedResults.sarif')); expect(info.dbItemPath).to.eq(Uri.file('/abc').fsPath); }); it('should check if interpreted results can be created', async () => { const info = createMockQueryInfo(true); - expect(info.canHaveInterpretedResults()).to.eq(true); + expect(info.queryEvalInfo.canHaveInterpretedResults(), '1').to.eq(true); - (info as any).databaseHasMetadataFile = false; - expect(info.canHaveInterpretedResults()).to.eq(false); + (info.queryEvalInfo as any).databaseHasMetadataFile = false; + expect(info.queryEvalInfo.canHaveInterpretedResults(), '2').to.eq(false); - (info as any).databaseHasMetadataFile = true; + (info.queryEvalInfo as any).databaseHasMetadataFile = true; info.metadata!.kind = undefined; - expect(info.canHaveInterpretedResults()).to.eq(false); + expect(info.queryEvalInfo.canHaveInterpretedResults(), '3').to.eq(false); info.metadata!.kind = 'table'; - expect(info.canHaveInterpretedResults()).to.eq(false); + expect(info.queryEvalInfo.canHaveInterpretedResults(), '4').to.eq(false); // Graphs are not interpreted unless canary is set info.metadata!.kind = 'graph'; - expect(info.canHaveInterpretedResults()).to.eq(false); + expect(info.queryEvalInfo.canHaveInterpretedResults(), '5').to.eq(false); (config.isCanary as sinon.SinonStub).returns(true); - expect(info.canHaveInterpretedResults()).to.eq(true); + expect(info.queryEvalInfo.canHaveInterpretedResults(), '6').to.eq(true); }); [SELECT_QUERY_NAME, 'other'].forEach(resultSetName => { it(`should export csv results for result set ${resultSetName}`, async () => { const csvLocation = path.join(tmpDir.name, 'test.csv'); - const qs = createMockQueryServerClient( - createMockCliServer({ - bqrsInfo: [{ 'result-sets': [{ name: resultSetName }, { name: 'hucairz' }] }], - bqrsDecode: [{ - columns: [{ kind: 'NotString' }, { kind: 'String' }], - tuples: [['a', 'b'], ['c', 'd']], - next: 1 - }, { - // just for fun, give a different set of columns here - // this won't happen with the real CLI, but it's a good test - columns: [{ kind: 'String' }, { kind: 'NotString' }, { kind: 'StillNotString' }], - tuples: [['a', 'b', 'c']] - }] - }) - ); + const cliServer = createMockCliServer({ + bqrsInfo: [{ 'result-sets': [{ name: resultSetName }, { name: 'hucairz' }] }], + bqrsDecode: [{ + columns: [{ kind: 'NotString' }, { kind: 'String' }], + tuples: [['a', 'b'], ['c', 'd']], + next: 1 + }, { + // just for fun, give a different set of columns here + // this won't happen with the real CLI, but it's a good test + columns: [{ kind: 'String' }, { kind: 'NotString' }, { kind: 'StillNotString' }], + tuples: [['a', 'b', 'c']] + }] + }); const info = createMockQueryInfo(); - const promise = info.exportCsvResults(qs, csvLocation); + const promise = info.queryEvalInfo.exportCsvResults(cliServer, csvLocation); const result = await promise; expect(result).to.eq(true); @@ -86,14 +86,14 @@ describe('run-queries', () => { expect(csv).to.eq('a,"b"\nc,"d"\n"a",b,c\n'); // now verify that we are using the expected result set - expect((qs.cliServer.bqrsDecode as sinon.SinonStub).callCount).to.eq(2); - expect((qs.cliServer.bqrsDecode as sinon.SinonStub).getCall(0).args[1]).to.eq(resultSetName); + expect((cliServer.bqrsDecode as sinon.SinonStub).callCount).to.eq(2); + expect((cliServer.bqrsDecode as sinon.SinonStub).getCall(0).args[1]).to.eq(resultSetName); }); }); it('should export csv results with characters that need to be escaped', async () => { const csvLocation = path.join(tmpDir.name, 'test.csv'); - const qs = createMockQueryServerClient( + const cliServer = createMockCliServer({ bqrsInfo: [{ 'result-sets': [{ name: SELECT_QUERY_NAME }, { name: 'hucairz' }] }], bqrsDecode: [{ @@ -109,10 +109,9 @@ describe('run-queries', () => { [123.98, 456.99], ], }] - }) - ); + }); const info = createMockQueryInfo(); - const promise = info.exportCsvResults(qs, csvLocation); + const promise = info.queryEvalInfo.exportCsvResults(cliServer, csvLocation); const result = await promise; expect(result).to.eq(true); @@ -121,19 +120,18 @@ describe('run-queries', () => { expect(csv).to.eq('"a","""b"""\nc,xxx,"d,yyy"\naaa " bbb,"ccc "" ddd"\ntrue,"false"\n123,"456"\n123.98,"456.99"\n'); // now verify that we are using the expected result set - expect((qs.cliServer.bqrsDecode as sinon.SinonStub).callCount).to.eq(1); - expect((qs.cliServer.bqrsDecode as sinon.SinonStub).getCall(0).args[1]).to.eq(SELECT_QUERY_NAME); + expect((cliServer.bqrsDecode as sinon.SinonStub).callCount).to.eq(1); + expect((cliServer.bqrsDecode as sinon.SinonStub).getCall(0).args[1]).to.eq(SELECT_QUERY_NAME); }); it('should handle csv exports for a query with no result sets', async () => { const csvLocation = path.join(tmpDir.name, 'test.csv'); - const qs = createMockQueryServerClient( + const cliServer = createMockCliServer({ bqrsInfo: [{ 'result-sets': [] }] - }) - ); + }); const info = createMockQueryInfo(); - const result = await info.exportCsvResults(qs, csvLocation); + const result = await info.queryEvalInfo.exportCsvResults(cliServer, csvLocation); expect(result).to.eq(false); }); @@ -187,9 +185,126 @@ describe('run-queries', () => { }); }); + + describe('register', () => { + it('should register', async () => { + const qs = createMockQueryServerClient( + { + cliConstraints: { + supportsDatabaseRegistration: () => true + } + } as any); + const runner = new LegacyQueryRunner(qs); + const mockProgress = 'progress-monitor'; + const mockCancel = 'cancel-token'; + const datasetUri = Uri.file('dataset-uri'); + + const dbItem: DatabaseItem = { + contents: { + datasetUri + } + } as any; + + await runner.registerDatabase(mockProgress as any, mockCancel as any, dbItem); + + expect(qs.sendRequest).to.have.been.calledOnceWith( + registerDatabases, + { + databases: [ + { + dbDir: datasetUri.fsPath, + workingSet: 'default' + } + ] + }, + mockCancel, + mockProgress + ); + }); + + it('should deregister', async () => { + const qs = createMockQueryServerClient( + { + cliConstraints: { + supportsDatabaseRegistration: () => true + } + } as any); + const runner = new LegacyQueryRunner(qs); + const mockProgress = 'progress-monitor'; + const mockCancel = 'cancel-token'; + const datasetUri = Uri.file('dataset-uri'); + + const dbItem: DatabaseItem = { + contents: { + datasetUri + } + } as any; + + await runner.deregisterDatabase(mockProgress as any, mockCancel as any, dbItem); + + expect(qs.sendRequest).to.have.been.calledOnceWith( + deregisterDatabases, + { + databases: [ + { + dbDir: datasetUri.fsPath, + workingSet: 'default' + } + ] + }, + mockCancel, + mockProgress + ); + }); + + it('should not register if unsupported', async () => { + const qs = createMockQueryServerClient( + { + cliConstraints: { + supportsDatabaseRegistration: () => false + } + } as any); + const runner = new LegacyQueryRunner(qs); + const mockProgress = 'progress-monitor'; + const mockCancel = 'cancel-token'; + const datasetUri = Uri.file('dataset-uri'); + + const dbItem: DatabaseItem = { + contents: { + datasetUri + } + } as any; + await runner.registerDatabase(mockProgress as any, mockCancel as any, dbItem); + expect(qs.sendRequest).not.to.have.been.called; + }); + + it('should not deregister if unsupported', async () => { + const qs = createMockQueryServerClient( + { + cliConstraints: { + supportsDatabaseRegistration: () => false + } + } as any); + const runner = new LegacyQueryRunner(qs); + const mockProgress = 'progress-monitor'; + const mockCancel = 'cancel-token'; + const datasetUri = Uri.file('dataset-uri'); + + const dbItem: DatabaseItem = { + contents: { + datasetUri + } + } as any; + await runner.registerDatabase(mockProgress as any, mockCancel as any, dbItem); + expect(qs.sendRequest).not.to.have.been.called; + }); + + }); + + let queryNum = 0; function createMockQueryInfo(databaseHasMetadataFile = true, saveDir = `save-dir${queryNum++}`) { - return new QueryEvaluationInfo( + return new QueryInProgress( saveDir, Uri.parse('file:///abc').fsPath, databaseHasMetadataFile,