mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #3094 from github/koesie10/remove-legacy-query-server

Browse Source 
Remove the legacy query server
This commit is contained in:
Koen Vlaswinkel
2023-12-05 12:08:54 +01:00
committed by GitHub
parent d28cc6e3f2 76534bccd7
commit 3020bf711b
15 changed files with 111 additions and 2436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
extensions/ql-vscode/src/codeql-cli/cli.ts
View File

@@ -2,7 +2,7 @@ import { EOL } from "os";
import { spawn } from "child-process-promise";
import * as child_process from "child_process";
import { readFile } from "fs-extra";
import { dirname, join, delimiter } from "path";
import { delimiter, dirname, join } from "path";
import * as sarif from "sarif";
import { SemVer } from "semver";
import { Readable } from "stream";
@@ -15,7 +15,7 @@ import {
DecodedBqrs,
DecodedBqrsChunk,
} from "../common/bqrs-cli-types";
import { allowCanaryQueryServer, CliConfig } from "../config";
import { CliConfig } from "../config";
import {
DistributionProvider,
FindDistributionResultKind,
@@ -1772,11 +1772,6 @@ export class CliVersionConstraint {
return (await this.cli.getVersion()).compare(v) >= 0;
}
async supportsNewQueryServer() {
// This allows users to explicitly opt-out of the new query server.
return allowCanaryQueryServer();
}
async supportsQlpacksKind() {
return this.isVersionAtLeast(
CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND,

14
extensions/ql-vscode/src/config.ts
View File

@@ -461,20 +461,6 @@ export function isCanary() {
return !!CANARY_FEATURES.getValue<boolean>();
}
/**
* Enables the experimental query server
*/
export const CANARY_QUERY_SERVER = new Setting(
"canaryQueryServer",
ROOT_SETTING,
);
// The default value for this setting is now `true`
export function allowCanaryQueryServer() {
const value = CANARY_QUERY_SERVER.getValue<boolean>();
return value === undefined ? true : !!value;
}
const LOG_INSIGHTS_SETTING = new Setting("logInsights", ROOT_SETTING);
export const JOIN_ORDER_WARNING_THRESHOLD = new Setting(
"joinOrderWarningThreshold",

50
extensions/ql-vscode/src/extension.ts
View File

@@ -13,7 +13,7 @@ import {
workspace,
} from "vscode";
import { LanguageClient } from "vscode-languageclient/node";
import { arch, platform, homedir } from "os";
import { arch, homedir, platform } from "os";
import { ensureDir } from "fs-extra";
import { join } from "path";
import { dirSync } from "tmp-promise";
@@ -35,13 +35,13 @@ import {
} from "./config";
import {
AstViewer,
install,
createIDEServer,
getQueryEditorCommands,
install,
TemplatePrintAstProvider,
TemplatePrintCfgProvider,
TemplateQueryDefinitionProvider,
TemplateQueryReferenceProvider,
createIDEServer,
} from "./language-support";
import { DatabaseManager } from "./databases/local-databases";
import { DatabaseUI } from "./databases/local-databases-ui";
@@ -68,15 +68,15 @@ import {
getErrorStack,
} from "./common/helpers-pure";
import {
ResultsView,
WebviewReveal,
LocalQueries,
QuickEvalCodeLensProvider,
ResultsView,
WebviewReveal,
} from "./local-queries";
import {
BaseLogger,
showAndLogExceptionWithTelemetry,
showAndLogErrorMessage,
showAndLogExceptionWithTelemetry,
showAndLogInformationMessage,
showAndLogWarningMessage,
} from "./common/logging";
@@ -88,10 +88,6 @@ import {
} from "./common/logging/vscode";
import { QueryHistoryManager } from "./query-history/query-history-manager";
import { CompletedLocalQueryInfo } from "./query-results";
import {
LegacyQueryRunner,
QueryServerClient as LegacyQueryServerClient,
} from "./query-server/legacy";
import { QLTestAdapterFactory } from "./query-testing/test-adapter";
import { TestUIService } from "./query-testing/test-ui";
import { CompareView } from "./compare/compare-view";
@@ -1242,29 +1238,17 @@ async function createQueryServer(
{ title: "CodeQL query server", location: ProgressLocation.Window },
task,
);
if (await cliServer.cliConstraints.supportsNewQueryServer()) {
const qs = new QueryServerClient(
app,
qlConfigurationListener,
cliServer,
qsOpts,
progressCallback,
);
ctx.subscriptions.push(qs);
await qs.startQueryServer();
return new NewQueryRunner(qs);
} else {
const qs = new LegacyQueryServerClient(
app,
qlConfigurationListener,
cliServer,
qsOpts,
progressCallback,
);
ctx.subscriptions.push(qs);
await qs.startQueryServer();
return new LegacyQueryRunner(qs);
}
const qs = new QueryServerClient(
app,
qlConfigurationListener,
cliServer,
qsOpts,
progressCallback,
);
ctx.subscriptions.push(qs);
await qs.startQueryServer();
return new NewQueryRunner(qs);
}
function getContextStoragePath(ctx: ExtensionContext) {

2
extensions/ql-vscode/src/query-results.ts
View File

@@ -22,8 +22,8 @@ import {
QueryOutputDir,
QueryWithResults,
} from "./run-queries-shared";
import { formatLegacyMessage } from "./query-server/legacy";
import { sarifParser } from "./common/sarif-parser";
import { formatLegacyMessage } from "./query-server/format-legacy-message";
/**
* query-results.ts

23
extensions/ql-vscode/src/query-server/format-legacy-message.ts Normal file
View File

@@ -0,0 +1,23 @@
import * as legacyMessages from "./legacy-messages";
// Used for formatting the result of a legacy query which might still be in the
// user's query history.
export function formatLegacyMessage(result: legacyMessages.EvaluationResult) {
switch (result.resultType) {
case legacyMessages.QueryResultType.CANCELLATION:
return `cancelled after ${Math.round(
result.evaluationTime / 1000,
)} seconds`;
case legacyMessages.QueryResultType.OOM:
return "out of memory";
case legacyMessages.QueryResultType.SUCCESS:
return `finished in ${Math.round(result.evaluationTime / 1000)} seconds`;
case legacyMessages.QueryResultType.TIMEOUT:
return `timed out after ${Math.round(
result.evaluationTime / 1000,
)} seconds`;
case legacyMessages.QueryResultType.OTHER_ERROR:
default:
return result.message ? `failed: ${result.message}` : "failed";
}
}

981
extensions/ql-vscode/src/query-server/legacy-messages.ts
View File

File diff suppressed because it is too large Load Diff

4
extensions/ql-vscode/src/query-server/legacy/index.ts
View File

@@ -1,4 +0,0 @@
export * from "./legacy-query-runner";
export * from "./query-server-client";
export * from "./run-queries";
export * from "./upgrades";

137
extensions/ql-vscode/src/query-server/legacy/legacy-query-runner.ts
View File

@@ -1,137 +0,0 @@
import { CancellationToken } from "vscode";
import { CodeQLCliServer } from "../../codeql-cli/cli";
import { ProgressCallback } from "../../common/vscode/progress";
import { Logger } from "../../common/logging";
import { DatabaseItem } from "../../databases/local-databases";
import {
Dataset,
deregisterDatabases,
registerDatabases,
} from "../legacy-messages";
import {
CoreQueryResults,
CoreQueryTarget,
QueryRunner,
} from "../query-runner";
import { QueryOutputDir } from "../../run-queries-shared";
import { QueryServerClient } from "./query-server-client";
import {
clearCacheInDatabase,
compileAndRunQueryAgainstDatabaseCore,
} from "./run-queries";
import { upgradeDatabaseExplicit } from "./upgrades";
export class LegacyQueryRunner extends QueryRunner {
constructor(public readonly qs: QueryServerClient) {
super();
}
get cliServer(): CodeQLCliServer {
return this.qs.cliServer;
}
get customLogDirectory(): string | undefined {
return undefined;
}
get logger(): Logger {
return this.qs.logger;
}
async restartQueryServer(
progress: ProgressCallback,
token: CancellationToken,
): Promise<void> {
await this.qs.restartQueryServer(progress, token);
}
onStart(
callBack: (
progress: ProgressCallback,
token: CancellationToken,
) => Promise<void>,
) {
this.qs.onDidStartQueryServer(callBack);
}
async clearCacheInDatabase(
dbItem: DatabaseItem,
token: CancellationToken,
): Promise<void> {
await clearCacheInDatabase(this.qs, dbItem, token);
}
async trimCacheInDatabase(
dbItem: DatabaseItem,
token: CancellationToken,
): Promise<void> {
// For the sake of conforming to the QueryRunner interface, delegate to clearCacheInDatabase
// just for the effect of getting a legacy query server deprecation error response.
await clearCacheInDatabase(this.qs, dbItem, token);
}
public async compileAndRunQueryAgainstDatabaseCore(
dbPath: string,
query: CoreQueryTarget,
additionalPacks: string[],
extensionPacks: string[] | undefined,
_additionalRunQueryArgs: Record<string, any>, // Ignored in legacy query server
generateEvalLog: boolean,
outputDir: QueryOutputDir,
progress: ProgressCallback,
token: CancellationToken,
templates: Record<string, string> | undefined,
logger: Logger,
): Promise<CoreQueryResults> {
return await compileAndRunQueryAgainstDatabaseCore(
this.qs,
dbPath,
query,
generateEvalLog,
additionalPacks,
extensionPacks,
outputDir,
progress,
token,
templates,
logger,
);
}
async deregisterDatabase(dbItem: DatabaseItem): Promise<void> {
if (dbItem.contents) {
const databases: Dataset[] = [
{
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: "default",
},
];
await this.qs.sendRequest(deregisterDatabases, { databases });
}
}
async registerDatabase(dbItem: DatabaseItem): Promise<void> {
if (dbItem.contents) {
const databases: Dataset[] = [
{
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: "default",
},
];
await this.qs.sendRequest(registerDatabases, { databases });
}
}
async upgradeDatabaseExplicit(
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
): Promise<void> {
await upgradeDatabaseExplicit(this.qs, dbItem, progress, token);
}
async clearPackCache(): Promise<void> {
/**
* Nothing needs to be done
*/
}
}

247
extensions/ql-vscode/src/query-server/legacy/query-server-client.ts
View File

@@ -1,247 +0,0 @@
import { ensureFile } from "fs-extra";
import { DisposableObject } from "../../common/disposable-object";
import { CancellationToken } from "vscode";
import { createMessageConnection, RequestType } from "vscode-jsonrpc/node";
import * as cli from "../../codeql-cli/cli";
import { QueryServerConfig } from "../../config";
import { Logger } from "../../common/logging";
import { ProgressReporter } from "../../common/logging/vscode";
import {
completeQuery,
EvaluationResult,
progress,
ProgressMessage,
WithProgressId,
} from "../legacy-messages";
import { ProgressCallback, ProgressTask } from "../../common/vscode/progress";
import { ServerProcess } from "../server-process";
import { App } from "../../common/app";
type WithProgressReporting = (
task: (
progress: ProgressReporter,
token: CancellationToken,
) => Thenable<void>,
) => Thenable<void>;
type ServerOpts = {
logger: Logger;
contextStoragePath: string;
};
/**
* Client that manages a query server process.
* The server process is started upon initialization and tracked during its lifetime.
* The server process is disposed when the client is disposed, or if the client asks
* to restart it (which disposes the existing process and starts a new one).
*/
export class QueryServerClient extends DisposableObject {
serverProcess?: ServerProcess;
evaluationResultCallbacks: { [key: number]: (res: EvaluationResult) => void };
progressCallbacks: {
[key: number]: ((res: ProgressMessage) => void) | undefined;
};
nextCallback: number;
nextProgress: number;
withProgressReporting: WithProgressReporting;
private readonly queryServerStartListeners = [] as Array<ProgressTask<void>>;
// Can't use standard vscode EventEmitter here since they do not cause the calling
// function to fail if one of the event handlers fail. This is something that
// we need here.
readonly onDidStartQueryServer = (e: ProgressTask<void>) => {
this.queryServerStartListeners.push(e);
};
public activeQueryLogger: Logger;
constructor(
app: App,
readonly config: QueryServerConfig,
readonly cliServer: cli.CodeQLCliServer,
readonly opts: ServerOpts,
withProgressReporting: WithProgressReporting,
) {
super();
// Since no query is active when we initialize, just point the "active query logger" to the
// default logger.
this.activeQueryLogger = this.logger;
// When the query server configuration changes, restart the query server.
if (config.onDidChangeConfiguration !== undefined) {
this.push(
config.onDidChangeConfiguration(() =>
app.commands.execute("codeQL.restartLegacyQueryServerOnConfigChange"),
),
);
}
this.withProgressReporting = withProgressReporting;
this.nextCallback = 0;
this.nextProgress = 0;
this.progressCallbacks = {};
this.evaluationResultCallbacks = {};
}
get logger(): Logger {
return this.opts.logger;
}
/** Stops the query server by disposing of the current server process. */
private stopQueryServer(): void {
if (this.serverProcess !== undefined) {
this.disposeAndStopTracking(this.serverProcess);
} else {
void this.logger.log("No server process to be stopped.");
}
}
/** Restarts the query server by disposing of the current server process and then starting a new one. */
async restartQueryServer(
progress: ProgressCallback,
token: CancellationToken,
): Promise<void> {
this.stopQueryServer();
await this.startQueryServer();
// Ensure we await all responses from event handlers so that
// errors can be properly reported to the user.
await Promise.all(
this.queryServerStartListeners.map((handler) => handler(progress, token)),
);
}
showLog(): void {
this.logger.show();
}
/** Starts a new query server process, sending progress messages to the status bar. */
async startQueryServer(): Promise<void> {
// Use an arrow function to preserve the value of `this`.
return this.withProgressReporting((progress, _) =>
this.startQueryServerImpl(progress),
);
}
/** Starts a new query server process, sending progress messages to the given reporter. */
private async startQueryServerImpl(
progressReporter: ProgressReporter,
): Promise<void> {
const ramArgs = await this.cliServer.resolveRam(
this.config.queryMemoryMb,
progressReporter,
);
const args = ["--threads", this.config.numThreads.toString()].concat(
ramArgs,
);
if (this.config.saveCache) {
args.push("--save-cache");
}
if (this.config.cacheSize > 0) {
args.push("--max-disk-cache");
args.push(this.config.cacheSize.toString());
}
args.push("--require-db-registration");
const structuredLogFile = `${this.opts.contextStoragePath}/structured-evaluator-log.json`;
await ensureFile(structuredLogFile);
args.push("--evaluator-log");
args.push(structuredLogFile);
if (this.config.debug) {
args.push("--debug", "--tuple-counting");
}
if (cli.shouldDebugQueryServer()) {
args.push(
"-J=-agentlib:jdwp=transport=dt_socket,address=localhost:9010,server=n,suspend=y,quiet=y",
);
}
const child = cli.spawnServer(
this.config.codeQlPath,
"CodeQL query server",
["execute", "query-server"],
args,
this.logger,
(data) =>
this.activeQueryLogger.log(data.toString(), {
trailingNewline: false,
}),
undefined, // no listener for stdout
progressReporter,
);
progressReporter.report({ message: "Connecting to CodeQL query server" });
const connection = createMessageConnection(child.stdout, child.stdin);
connection.onRequest(completeQuery, (res) => {
if (!(res.runId in this.evaluationResultCallbacks)) {
void this.logger.log(
`No callback associated with run id ${res.runId}, continuing without executing any callback`,
);
} else {
this.evaluationResultCallbacks[res.runId](res);
}
return {};
});
connection.onNotification(progress, (res) => {
const callback = this.progressCallbacks[res.id];
if (callback) {
callback(res);
}
});
this.serverProcess = new ServerProcess(
child,
connection,
"Query server",
this.logger,
);
// Ensure the server process is disposed together with this client.
this.track(this.serverProcess);
connection.listen();
progressReporter.report({ message: "Connected to CodeQL query server" });
this.nextCallback = 0;
this.nextProgress = 0;
this.progressCallbacks = {};
this.evaluationResultCallbacks = {};
}
registerCallback(callback: (res: EvaluationResult) => void): number {
const id = this.nextCallback++;
this.evaluationResultCallbacks[id] = callback;
return id;
}
unRegisterCallback(id: number): void {
delete this.evaluationResultCallbacks[id];
}
get serverProcessPid(): number {
return this.serverProcess!.child.pid || 0;
}
async sendRequest<P, R, E>(
type: RequestType<WithProgressId<P>, R, E>,
parameter: P,
token?: CancellationToken,
progress?: (res: ProgressMessage) => void,
): Promise<R> {
const id = this.nextProgress++;
this.progressCallbacks[id] = progress;
try {
if (this.serverProcess === undefined) {
throw new Error("No query server process found.");
}
return await this.serverProcess.connection.sendRequest(
type,
{ body: parameter, progressId: id },
token,
);
} finally {
delete this.progressCallbacks[id];
}
}
}

533
extensions/ql-vscode/src/query-server/legacy/run-queries.ts
View File

@@ -1,533 +0,0 @@
import * as tmp from "tmp-promise";
import { basename, join } from "path";
import { CancellationToken, Uri } from "vscode";
import { LSPErrorCodes, ResponseError } from "vscode-languageclient";
import * as cli from "../../codeql-cli/cli";
import {
DatabaseContentsWithDbScheme,
DatabaseItem,
DatabaseResolver,
} from "../../databases/local-databases";
import { tmpDir } from "../../tmp-dir";
import { ProgressCallback } from "../../common/vscode/progress";
import { QueryMetadata } from "../../common/interface-types";
import { extLogger } from "../../common/logging/vscode";
import {
Logger,
showAndLogExceptionWithTelemetry,
showAndLogWarningMessage,
} from "../../common/logging";
import * as messages from "../legacy-messages";
import * as newMessages from "../new-messages";
import * as qsClient from "./query-server-client";
import { asError, getErrorMessage } from "../../common/helpers-pure";
import { compileDatabaseUpgradeSequence } from "./upgrades";
import { QueryEvaluationInfo, QueryOutputDir } from "../../run-queries-shared";
import { redactableError } from "../../common/errors";
import { CoreQueryResults, CoreQueryTarget } from "../query-runner";
import { Position } from "../messages-shared";
import { ensureDirSync } from "fs-extra";
import { telemetryListener } from "../../common/vscode/telemetry";
const upgradesTmpDir = join(tmpDir.name, "upgrades");
ensureDirSync(upgradesTmpDir);
export async function compileQuery(
qs: qsClient.QueryServerClient,
program: messages.QlProgram,
quickEvalPosition: Position | undefined,
outputDir: QueryOutputDir,
progress: ProgressCallback,
token: CancellationToken,
logger: Logger,
): Promise<messages.CompilationMessage[]> {
let compiled: messages.CheckQueryResult | undefined;
try {
const target: messages.CompilationTarget = quickEvalPosition
? {
quickEval: { quickEvalPos: quickEvalPosition },
}
: { query: {} };
const params: messages.CompileQueryParams = {
compilationOptions: {
computeNoLocationUrls: true,
failOnWarnings: false,
fastCompilation: false,
includeDilInQlo: true,
localChecking: false,
noComputeGetUrl: false,
noComputeToString: false,
computeDefaultStrings: true,
emitDebugInfo: true,
},
extraOptions: {
timeoutSecs: qs.config.timeoutSecs,
},
queryToCheck: program,
resultPath: outputDir.compileQueryPath,
target,
};
// Update the active query logger every time there is a new request to compile.
// This isn't ideal because in situations where there are queries running
// in parallel, each query's log messages are interleaved. Fixing this
// properly will require a change in the query server.
qs.activeQueryLogger = logger;
compiled = await qs.sendRequest(
messages.compileQuery,
params,
token,
progress,
);
} finally {
void logger.log(" - - - COMPILATION DONE - - - ");
}
return (compiled?.messages || []).filter(
(msg) => msg.severity === messages.Severity.ERROR,
);
}
async function runQuery(
qs: qsClient.QueryServerClient,
upgradeQlo: string | undefined,
availableMlModels: cli.MlModelInfo[],
dbContents: DatabaseContentsWithDbScheme,
templates: Record<string, string> | undefined,
generateEvalLog: boolean,
outputDir: QueryOutputDir,
progress: ProgressCallback,
token: CancellationToken,
): Promise<messages.EvaluationResult> {
let result: messages.EvaluationResult | null = null;
const logPath = outputDir.logPath;
const callbackId = qs.registerCallback((res) => {
result = {
...res,
logFileLocation: logPath,
};
});
const availableMlModelUris: messages.MlModel[] = availableMlModels.map(
(model) => ({ uri: Uri.file(model.path).toString(true) }),
);
const queryToRun: messages.QueryToRun = {
resultsPath: outputDir.bqrsPath,
qlo: Uri.file(outputDir.compileQueryPath).toString(),
compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(),
allowUnknownTemplates: true,
templateValues: createSimpleTemplates(templates),
availableMlModels: availableMlModelUris,
id: callbackId,
timeoutSecs: qs.config.timeoutSecs,
};
const dataset: messages.Dataset = {
dbDir: dbContents.datasetUri.fsPath,
workingSet: "default",
};
if (generateEvalLog) {
await qs.sendRequest(messages.startLog, {
db: dataset,
logPath: outputDir.evalLogPath,
});
}
const params: messages.EvaluateQueriesParams = {
db: dataset,
evaluateId: callbackId,
queries: [queryToRun],
stopOnError: false,
useSequenceHint: false,
};
try {
await qs.sendRequest(messages.runQueries, params, token, progress);
} finally {
qs.unRegisterCallback(callbackId);
if (generateEvalLog) {
await qs.sendRequest(messages.endLog, {
db: dataset,
logPath: outputDir.evalLogPath,
});
}
}
return (
result || {
evaluationTime: 0,
message: "No result from server",
queryId: -1,
runId: callbackId,
resultType: messages.QueryResultType.OTHER_ERROR,
}
);
}
/**
* A collection of evaluation-time information about a query,
* including the query itself, and where we have decided to put
* temporary files associated with it, such as the compiled query
* output and results.
*/
export class QueryInProgress {
public queryEvalInfo: QueryEvaluationInfo;
constructor(
readonly querySaveDir: string,
readonly dbItemPath: string,
databaseHasMetadataFile: boolean,
readonly queryDbscheme: string, // the dbscheme file the query expects, ba`sed on library path resolution
readonly quickEvalPosition?: messages.Position,
readonly metadata?: QueryMetadata,
readonly templates?: Record<string, string>,
) {
this.queryEvalInfo = new QueryEvaluationInfo(
querySaveDir,
dbItemPath,
databaseHasMetadataFile,
quickEvalPosition,
metadata,
);
/**/
}
}
export async function clearCacheInDatabase(
qs: qsClient.QueryServerClient,
dbItem: DatabaseItem,
token: CancellationToken,
): Promise<messages.ClearCacheResult> {
if (dbItem.contents === undefined) {
throw new Error("Can't clear the cache in an invalid database.");
}
const db: messages.Dataset = {
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: "default",
};
const params: messages.ClearCacheParams = {
dryRun: false,
db,
};
return qs.sendRequest(messages.clearCache, params, token);
}
function reportNoUpgradePath(
qlProgram: messages.QlProgram,
queryDbscheme: string,
): void {
throw new Error(
`Query ${qlProgram.queryPath} expects database scheme ${queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.`,
);
}
/**
* Compile a non-destructive upgrade.
*/
async function compileNonDestructiveUpgrade(
qs: qsClient.QueryServerClient,
upgradeTemp: tmp.DirectoryResult,
queryDbscheme: string,
qlProgram: messages.QlProgram,
dbContents: DatabaseContentsWithDbScheme,
progress: ProgressCallback,
token: CancellationToken,
): Promise<string> {
// Dependencies may exist outside of the workspace and they are always on the resolved search path.
const upgradesPath = qlProgram.libraryPath;
const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(
dbContents.dbSchemeUri.fsPath,
upgradesPath,
true,
queryDbscheme,
);
if (!matchesTarget) {
reportNoUpgradePath(qlProgram, queryDbscheme);
}
const result = await compileDatabaseUpgradeSequence(
qs,
scripts,
upgradeTemp,
progress,
token,
);
if (result.compiledUpgrade === undefined) {
const error = result.error || "[no error message available]";
throw new Error(error);
}
// We can upgrade to the actual target
qlProgram.dbschemePath = queryDbscheme;
// We are new enough that we will always support single file upgrades.
return result.compiledUpgrade;
}
function translateLegacyResult(
legacyResult: messages.EvaluationResult,
): Omit<CoreQueryResults, "dispose"> {
let newResultType: newMessages.QueryResultType;
let newMessage = legacyResult.message;
switch (legacyResult.resultType) {
case messages.QueryResultType.SUCCESS:
newResultType = newMessages.QueryResultType.SUCCESS;
break;
case messages.QueryResultType.CANCELLATION:
newResultType = newMessages.QueryResultType.CANCELLATION;
break;
case messages.QueryResultType.OOM:
newResultType = newMessages.QueryResultType.OOM;
break;
case messages.QueryResultType.TIMEOUT:
// This is the only legacy result type that doesn't exist for the new query server. Format the
// message here, and let the later code treat it as `OTHER_ERROR`.
newResultType = newMessages.QueryResultType.OTHER_ERROR;
newMessage = `timed out after ${Math.round(
legacyResult.evaluationTime / 1000,
)} seconds`;
break;
case messages.QueryResultType.OTHER_ERROR:
default:
newResultType = newMessages.QueryResultType.OTHER_ERROR;
break;
}
return {
resultType: newResultType,
message: newMessage,
evaluationTime: legacyResult.evaluationTime,
};
}
export async function compileAndRunQueryAgainstDatabaseCore(
qs: qsClient.QueryServerClient,
dbPath: string,
query: CoreQueryTarget,
generateEvalLog: boolean,
additionalPacks: string[],
extensionPacks: string[] | undefined,
outputDir: QueryOutputDir,
progress: ProgressCallback,
token: CancellationToken,
templates: Record<string, string> | undefined,
logger: Logger,
): Promise<CoreQueryResults> {
if (extensionPacks !== undefined && extensionPacks.length > 0) {
void showAndLogWarningMessage(
extLogger,
"Legacy query server does not support extension packs.",
);
}
const dbContents = await DatabaseResolver.resolveDatabaseContents(
Uri.file(dbPath),
);
// Figure out the library path for the query.
const packConfig = await qs.cliServer.resolveLibraryPath(
additionalPacks,
query.queryPath,
);
if (!packConfig.dbscheme) {
throw new Error(
"Could not find a database scheme for this query. Please check that you have a valid qlpack.yml or codeql-pack.yml file for this query, which refers to a database scheme either in the `dbscheme` field or through one of its dependencies.",
);
}
// Check whether the query has an entirely different schema from the
// database. (Queries that merely need the database to be upgraded
// won't trigger this check)
// This test will produce confusing results if we ever change the name of the database schema files.
const querySchemaName = basename(packConfig.dbscheme);
const dbSchemaName = basename(dbContents.dbSchemeUri?.fsPath);
if (querySchemaName !== dbSchemaName) {
void extLogger.log(
`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`,
);
throw new Error(
`The query ${basename(
query.queryPath,
)} cannot be run against the selected database: their target languages are different. Please select a different database and try again.`,
);
}
const qlProgram: messages.QlProgram = {
// The project of the current document determines which library path
// we use. The `libraryPath` field in this server message is relative
// to the workspace root, not to the project root.
libraryPath: packConfig.libraryPath,
// Since we are compiling and running a query against a database,
// we use the database's DB scheme here instead of the DB scheme
// from the current document's project.
dbschemePath: dbContents.dbSchemeUri.fsPath,
queryPath: query.queryPath,
};
let availableMlModels: cli.MlModelInfo[] = [];
try {
availableMlModels = (
await qs.cliServer.resolveMlModels(additionalPacks, query.queryPath)
).models;
if (availableMlModels.length) {
void extLogger.log(
`Found available ML models at the following paths: ${availableMlModels
.map((x) => `'${x.path}'`)
.join(", ")}.`,
);
} else {
void extLogger.log("Did not find any available ML models.");
}
} catch (e) {
void showAndLogExceptionWithTelemetry(
extLogger,
telemetryListener,
redactableError(
asError(e),
)`Couldn't resolve available ML models for ${qlProgram.queryPath}. Running the query without any ML models: ${e}.`,
);
}
let upgradeDir: tmp.DirectoryResult | undefined;
try {
upgradeDir = await tmp.dir({ dir: upgradesTmpDir, unsafeCleanup: true });
const upgradeQlo = await compileNonDestructiveUpgrade(
qs,
upgradeDir,
packConfig.dbscheme,
qlProgram,
dbContents,
progress,
token,
);
let errors;
try {
errors = await compileQuery(
qs,
qlProgram,
query.quickEvalPosition,
outputDir,
progress,
token,
logger,
);
} catch (e) {
if (
e instanceof ResponseError &&
e.code === LSPErrorCodes.RequestCancelled
) {
return createSyntheticResult("Query cancelled");
} else {
throw e;
}
}
if (errors.length === 0) {
const result = await runQuery(
qs,
upgradeQlo,
availableMlModels,
dbContents,
templates,
generateEvalLog,
outputDir,
progress,
token,
);
if (result.resultType !== messages.QueryResultType.SUCCESS) {
const error = result.message
? redactableError`${result.message}`
: redactableError`Failed to run query`;
void showAndLogExceptionWithTelemetry(
extLogger,
telemetryListener,
error,
);
}
return translateLegacyResult(result);
} else {
// Error dialogs are limited in size and scrollability,
// so we include a general description of the problem,
// and direct the user to the output window for the detailed compilation messages.
// However we don't show quick eval errors there so we need to display them anyway.
void logger.log(
`Failed to compile query ${query.queryPath} against database scheme ${qlProgram.dbschemePath}:`,
);
const formattedMessages: string[] = [];
for (const error of errors) {
const message = error.message || "[no error message available]";
const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
formattedMessages.push(formatted);
void logger.log(formatted);
}
return {
evaluationTime: 0,
resultType: newMessages.QueryResultType.COMPILATION_ERROR,
message: formattedMessages[0],
};
}
} finally {
try {
await upgradeDir?.cleanup();
} catch (e) {
void logger.log(
`Could not clean up the upgrades dir. Reason: ${getErrorMessage(e)}`,
);
}
}
}
export function formatLegacyMessage(result: messages.EvaluationResult) {
switch (result.resultType) {
case messages.QueryResultType.CANCELLATION:
return `cancelled after ${Math.round(
result.evaluationTime / 1000,
)} seconds`;
case messages.QueryResultType.OOM:
return "out of memory";
case messages.QueryResultType.SUCCESS:
return `finished in ${Math.round(result.evaluationTime / 1000)} seconds`;
case messages.QueryResultType.TIMEOUT:
return `timed out after ${Math.round(
result.evaluationTime / 1000,
)} seconds`;
case messages.QueryResultType.OTHER_ERROR:
default:
return result.message ? `failed: ${result.message}` : "failed";
}
}
/**
* Create a synthetic result for a query that failed to compile.
*/
function createSyntheticResult(message: string): CoreQueryResults {
return {
evaluationTime: 0,
resultType: newMessages.QueryResultType.OTHER_ERROR,
message,
};
}
function createSimpleTemplates(
templates: Record<string, string> | undefined,
): messages.TemplateDefinitions | undefined {
if (!templates) {
return undefined;
}
const result: messages.TemplateDefinitions = {};
for (const key of Object.keys(templates)) {
result[key] = {
values: {
tuples: [[{ stringValue: templates[key] }]],
},
};
}
return result;
}

298
extensions/ql-vscode/src/query-server/legacy/upgrades.ts
View File

@@ -1,298 +0,0 @@
import * as vscode from "vscode";
import { tmpDir } from "../../tmp-dir";
import { getOnDiskWorkspaceFolders } from "../../common/vscode/workspace-folders";
import {
ProgressCallback,
UserCancellationException,
} from "../../common/vscode/progress";
import { extLogger } from "../../common/logging/vscode";
import { showAndLogExceptionWithTelemetry } from "../../common/logging";
import * as messages from "../legacy-messages";
import * as qsClient from "./query-server-client";
import * as tmp from "tmp-promise";
import { dirname } from "path";
import { DatabaseItem } from "../../databases/local-databases";
import { asError, getErrorMessage } from "../../common/helpers-pure";
import { redactableError } from "../../common/errors";
import { telemetryListener } from "../../common/vscode/telemetry";
/**
* Maximum number of lines to include from database upgrade message,
* to work around the fact that we can't guarantee a scrollable text
* box for it when displaying in dialog boxes.
*/
const MAX_UPGRADE_MESSAGE_LINES = 10;
/**
* Compile a database upgrade sequence.
* Callers must check that this is valid with the current queryserver first.
*/
export async function compileDatabaseUpgradeSequence(
qs: qsClient.QueryServerClient,
resolvedSequence: string[],
currentUpgradeTmp: tmp.DirectoryResult,
progress: ProgressCallback,
token: vscode.CancellationToken,
): Promise<messages.CompileUpgradeSequenceResult> {
// If possible just compile the upgrade sequence
return await qs.sendRequest(
messages.compileUpgradeSequence,
{
upgradeTempDir: currentUpgradeTmp.path,
upgradePaths: resolvedSequence,
},
token,
progress,
);
}
async function compileDatabaseUpgrade(
qs: qsClient.QueryServerClient,
dbItem: DatabaseItem,
targetDbScheme: string,
resolvedSequence: string[],
currentUpgradeTmp: tmp.DirectoryResult,
progress: ProgressCallback,
token: vscode.CancellationToken,
): Promise<messages.CompileUpgradeResult> {
if (!dbItem.contents?.dbSchemeUri) {
throw new Error("Database is invalid, and cannot be upgraded.");
}
// We have the upgrades we want but compileUpgrade
// requires searching for them. So we use the parent directories of the upgrades
// as the upgrade path.
const parentDirs = resolvedSequence.map((dir) => dirname(dir));
const uniqueParentDirs = new Set(parentDirs);
progress({
step: 1,
maxStep: 3,
message: "Checking for database upgrades",
});
return qs.sendRequest(
messages.compileUpgrade,
{
upgrade: {
fromDbscheme: dbItem.contents.dbSchemeUri.fsPath,
toDbscheme: targetDbScheme,
additionalUpgrades: Array.from(uniqueParentDirs),
},
upgradeTempDir: currentUpgradeTmp.path,
singleFileUpgrades: true,
},
token,
progress,
);
}
/**
* Checks whether the user wants to proceed with the upgrade.
* Reports errors to both the user and the console.
*/
async function checkAndConfirmDatabaseUpgrade(
compiled: messages.CompiledUpgrades,
db: DatabaseItem,
quiet: boolean,
): Promise<void> {
let descriptionMessage = "";
const descriptions = getUpgradeDescriptions(compiled);
for (const script of descriptions) {
descriptionMessage += `Would perform upgrade: ${script.description}\n`;
descriptionMessage += `\t-> Compatibility: ${script.compatibility}\n`;
}
void extLogger.log(descriptionMessage);
// If the quiet flag is set, do the upgrade without a popup.
if (quiet) {
return;
}
// Ask the user to confirm the upgrade.
const showLogItem: vscode.MessageItem = {
title: "No, Show Changes",
isCloseAffordance: true,
};
const yesItem = { title: "Yes", isCloseAffordance: false };
const noItem = { title: "No", isCloseAffordance: true };
const dialogOptions: vscode.MessageItem[] = [yesItem, noItem];
let messageLines = descriptionMessage.split("\n");
if (messageLines.length > MAX_UPGRADE_MESSAGE_LINES) {
messageLines = messageLines.slice(0, MAX_UPGRADE_MESSAGE_LINES);
messageLines.push(
'The list of upgrades was truncated, click "No, Show Changes" to see the full list.',
);
dialogOptions.push(showLogItem);
}
const message = `Should the database ${
db.databaseUri.fsPath
} be upgraded?\n\n${messageLines.join("\n")}`;
const chosenItem = await vscode.window.showInformationMessage(
message,
{ modal: true },
...dialogOptions,
);
if (chosenItem === showLogItem) {
extLogger.outputChannel.show();
}
if (chosenItem !== yesItem) {
throw new UserCancellationException("User cancelled the database upgrade.");
}
}
/**
* Get the descriptions from a compiled upgrade
*/
function getUpgradeDescriptions(
compiled: messages.CompiledUpgrades,
): messages.UpgradeDescription[] {
// We use the presence of compiledUpgradeFile to check
// if it is multifile or not. We need to explicitly check undefined
// as the types claim the empty string is a valid value
if (compiled.compiledUpgradeFile === undefined) {
return compiled.scripts.map((script) => script.description);
} else {
return compiled.descriptions;
}
}
/**
* Command handler for 'Upgrade Database'.
* Attempts to upgrade the given database to the given target DB scheme, using the given directory of upgrades.
* First performs a dry-run and prompts the user to confirm the upgrade.
* Reports errors during compilation and evaluation of upgrades to the user.
*/
export async function upgradeDatabaseExplicit(
qs: qsClient.QueryServerClient,
dbItem: DatabaseItem,
progress: ProgressCallback,
token: vscode.CancellationToken,
): Promise<messages.RunUpgradeResult | undefined> {
const searchPath: string[] = getOnDiskWorkspaceFolders();
if (!dbItem?.contents?.dbSchemeUri) {
throw new Error("Database is invalid, and cannot be upgraded.");
}
const upgradeInfo = await qs.cliServer.resolveUpgrades(
dbItem.contents.dbSchemeUri.fsPath,
searchPath,
false,
);
const { scripts, finalDbscheme } = upgradeInfo;
if (finalDbscheme === undefined) {
throw new Error("Could not determine target dbscheme to upgrade to.");
}
const currentUpgradeTmp = await tmp.dir({
dir: tmpDir.name,
prefix: "upgrade_",
keep: false,
unsafeCleanup: true,
});
try {
let compileUpgradeResult: messages.CompileUpgradeResult;
try {
compileUpgradeResult = await compileDatabaseUpgrade(
qs,
dbItem,
finalDbscheme,
scripts,
currentUpgradeTmp,
progress,
token,
);
} catch (e) {
void showAndLogExceptionWithTelemetry(
extLogger,
telemetryListener,
redactableError(
asError(e),
)`Compilation of database upgrades failed: ${getErrorMessage(e)}`,
);
return;
} finally {
void qs.logger.log("Done compiling database upgrade.");
}
if (!compileUpgradeResult.compiledUpgrades) {
const error = compileUpgradeResult.error
? redactableError`${compileUpgradeResult.error}`
: redactableError`[no error message available]`;
void showAndLogExceptionWithTelemetry(
extLogger,
telemetryListener,
redactableError`Compilation of database upgrades failed: ${error}`,
);
return;
}
await checkAndConfirmDatabaseUpgrade(
compileUpgradeResult.compiledUpgrades,
dbItem,
qs.cliServer.quiet,
);
try {
void qs.logger.log("Running the following database upgrade:");
getUpgradeDescriptions(compileUpgradeResult.compiledUpgrades)
.map((s) => s.description)
.join("\n");
const result = await runDatabaseUpgrade(
qs,
dbItem,
compileUpgradeResult.compiledUpgrades,
progress,
token,
);
// TODO Can remove the next lines when https://github.com/github/codeql-team/issues/1241 is fixed
// restart the query server to avoid a bug in the CLI where the upgrade is applied, but the old dbscheme
// is still cached in memory.
await qs.restartQueryServer(progress, token);
return result;
} catch (e) {
void showAndLogExceptionWithTelemetry(
extLogger,
telemetryListener,
redactableError(asError(e))`Database upgrade failed: ${getErrorMessage(
e,
)}`,
);
return;
} finally {
void qs.logger.log("Done running database upgrade.");
}
} finally {
await currentUpgradeTmp.cleanup();
}
}
async function runDatabaseUpgrade(
qs: qsClient.QueryServerClient,
db: DatabaseItem,
upgrades: messages.CompiledUpgrades,
progress: ProgressCallback,
token: vscode.CancellationToken,
): Promise<messages.RunUpgradeResult> {
if (db.contents === undefined || db.contents.datasetUri === undefined) {
throw new Error("Can't upgrade an invalid database.");
}
const database: messages.Dataset = {
dbDir: db.contents.datasetUri.fsPath,
workingSet: "default",
};
const params: messages.RunUpgradeParams = {
db: database,
timeoutSecs: qs.config.timeoutSecs,
toRun: upgrades,
};
return qs.sendRequest(messages.runUpgrade, params, token, progress);
}

15
extensions/ql-vscode/src/run-queries-shared.ts
View File

@@ -220,17 +220,10 @@ export class QueryEvaluationInfo extends QueryOutputDir {
}
const compiledQuery = this.compileQueryPath;
if (!(await pathExists(compiledQuery))) {
if (await cliServer.cliConstraints.supportsNewQueryServer()) {
// This could be from the new query server
// in which case we expect the qlo to be missing so we should ignore it
throw new Error(
`DIL was not found. Expected location: '${this.dilPath}'`,
);
} else {
throw new Error(
`Cannot create DIL because compiled query is missing. ${compiledQuery}`,
);
}
// We expect the qlo to be missing so we should ignore it
throw new Error(
`DIL was not found. Expected location: '${this.dilPath}'`,
);
}
await cliServer.generateDil(compiledQuery, this.dilPath);

46
extensions/ql-vscode/test/vscode-tests/no-workspace/query-history/store/query-history-store.test.ts
View File

@@ -9,6 +9,7 @@ import {
InitialQueryInfo,
} from "../../../../../src/query-results";
import {
QueryEvaluationInfo,
QueryOutputDir,
QueryWithResults,
} from "../../../../../src/run-queries-shared";
@@ -16,7 +17,6 @@ import { DatabaseInfo } from "../../../../../src/common/interface-types";
import { CancellationTokenSource, Uri } from "vscode";
import { tmpDir } from "../../../../../src/tmp-dir";
import { QueryResultType } from "../../../../../src/query-server/legacy-messages";
import { QueryInProgress } from "../../../../../src/query-server/legacy";
import { VariantAnalysisHistoryItem } from "../../../../../src/query-history/variant-analysis-history-item";
import { QueryHistoryInfo } from "../../../../../src/query-history/query-history-info";
import { createMockVariantAnalysisHistoryItem } from "../../../../factories/query-history/variant-analysis-history-item";
@@ -42,34 +42,16 @@ describe("write and read", () => {
infoSuccessRaw = createMockFullQueryInfo(
"a",
createMockQueryWithResults(
`${queryPath}-a`,
false,
false,
"/a/b/c/a",
false,
),
createMockQueryWithResults(`${queryPath}-a`, false, "/a/b/c/a"),
);
infoSuccessInterpreted = createMockFullQueryInfo(
"b",
createMockQueryWithResults(
`${queryPath}-b`,
true,
true,
"/a/b/c/b",
false,
),
createMockQueryWithResults(`${queryPath}-b`, true, "/a/b/c/b"),
);
infoEarlyFailure = createMockFullQueryInfo("c", undefined, true);
infoLateFailure = createMockFullQueryInfo(
"d",
createMockQueryWithResults(
`${queryPath}-c`,
false,
false,
"/a/b/c/d",
false,
),
createMockQueryWithResults(`${queryPath}-c`, false, "/a/b/c/d"),
);
infoInProgress = createMockFullQueryInfo("e");
@@ -138,13 +120,7 @@ describe("write and read", () => {
const queryItem = createMockFullQueryInfo(
"a",
createMockQueryWithResults(
`${queryPath}-a`,
false,
false,
"/a/b/c/a",
false,
),
createMockQueryWithResults(`${queryPath}-a`, false, "/a/b/c/a"),
false,
null,
);
@@ -277,20 +253,17 @@ describe("write and read", () => {
function createMockQueryWithResults(
queryPath: string,
didRunSuccessfully = true,
hasInterpretedResults = true,
dbPath = "/a/b/c",
includeSpies = true,
): QueryWithResults {
// pretend that the results path exists
const resultsPath = join(queryPath, "results.bqrs");
mkdirpSync(queryPath);
writeFileSync(resultsPath, "", "utf8");
const query = new QueryInProgress(
const queryEvalInfo = new QueryEvaluationInfo(
queryPath,
Uri.file(dbPath).fsPath,
true,
"queryDbscheme",
undefined,
{
name: "vwx",
@@ -298,7 +271,7 @@ describe("write and read", () => {
);
const result: QueryWithResults = {
query: query.queryEvalInfo,
query: queryEvalInfo,
successful: didRunSuccessfully,
message: "foo",
result: {
@@ -309,11 +282,6 @@ describe("write and read", () => {
},
};
if (includeSpies) {
(query as any).hasInterpretedResults = () =>
Promise.resolve(hasInterpretedResults);
}
return result;
}
});

18
extensions/ql-vscode/test/vscode-tests/no-workspace/query-results.test.ts
View File

@@ -13,6 +13,7 @@ import {
interpretResultsSarif,
} from "../../../src/query-results";
import {
QueryEvaluationInfo,
QueryOutputDir,
QueryWithResults,
} from "../../../src/run-queries-shared";
@@ -24,16 +25,13 @@ import {
import { CodeQLCliServer, SourceInfo } from "../../../src/codeql-cli/cli";
import { CancellationTokenSource, Uri } from "vscode";
import { tmpDir } from "../../../src/tmp-dir";
import {
formatLegacyMessage,
QueryInProgress,
} from "../../../src/query-server/legacy/run-queries";
import {
EvaluationResult,
QueryResultType,
} from "../../../src/query-server/legacy-messages";
import { sleep } from "../../../src/common/time";
import { mockedObject } from "../utils/mocking.helpers";
import { formatLegacyMessage } from "../../../src/query-server/format-legacy-message";
describe("query-results", () => {
let queryPath: string;
@@ -452,20 +450,17 @@ describe("query-results", () => {
function createMockQueryWithResults(
queryPath: string,
didRunSuccessfully = true,
hasInterpretedResults = true,
dbPath = "/a/b/c",
includeSpies = true,
): QueryWithResults {
// pretend that the results path exists
const resultsPath = join(queryPath, "results.bqrs");
mkdirpSync(queryPath);
writeFileSync(resultsPath, "", "utf8");
const query = new QueryInProgress(
const queryEvalInfo = new QueryEvaluationInfo(
queryPath,
Uri.file(dbPath).fsPath,
true,
"queryDbscheme",
undefined,
{
name: "vwx",
@@ -473,7 +468,7 @@ describe("query-results", () => {
);
const result: QueryWithResults = {
query: query.queryEvalInfo,
query: queryEvalInfo,
successful: didRunSuccessfully,
message: "foo",
result: {
@@ -484,11 +479,6 @@ describe("query-results", () => {
},
};
if (includeSpies) {
(query as any).hasInterpretedResults = () =>
Promise.resolve(hasInterpretedResults);
}
return result;
}

170
extensions/ql-vscode/test/vscode-tests/no-workspace/run-queries.test.ts
View File

@@ -2,27 +2,22 @@ import { join } from "path";
import { readFileSync } from "fs-extra";
import { Uri } from "vscode";
import {
Severity,
compileQuery,
registerDatabases,
deregisterDatabases,
} from "../../../src/query-server/legacy-messages";
import * as config from "../../../src/config";
import { tmpDir } from "../../../src/tmp-dir";
import { CodeQLCliServer } from "../../../src/codeql-cli/cli";
import { SELECT_QUERY_NAME } from "../../../src/language-support";
import {
QueryInProgress,
compileQuery as compileQueryLegacy,
} from "../../../src/query-server/legacy/run-queries";
import {
LegacyQueryRunner,
QueryServerClient,
} from "../../../src/query-server/legacy";
import { DatabaseItem } from "../../../src/databases/local-databases";
import { DeepPartial, mockedObject } from "../utils/mocking.helpers";
DeepPartial,
mockDatabaseItem,
mockedObject,
} from "../utils/mocking.helpers";
import { BqrsKind } from "../../../src/common/bqrs-cli-types";
import { NewQueryRunner, QueryServerClient } from "../../../src/query-server";
import { QueryEvaluationInfo } from "../../../src/run-queries-shared";
import {
deregisterDatabases,
registerDatabases,
} from "../../../src/query-server/new-messages";
describe("run-queries", () => {
let isCanarySpy: jest.SpiedFunction<typeof config.isCanary>;
@@ -33,45 +28,45 @@ describe("run-queries", () => {
it("should create a QueryEvaluationInfo", () => {
const saveDir = "query-save-dir";
const info = createMockQueryInfo(true, saveDir);
const queryEvalInfo = createMockQueryEvaluationInfo(true, saveDir);
expect(info.queryEvalInfo.dilPath).toBe(join(saveDir, "results.dil"));
expect(info.queryEvalInfo.resultsPaths.resultsPath).toBe(
expect(queryEvalInfo.dilPath).toBe(join(saveDir, "results.dil"));
expect(queryEvalInfo.resultsPaths.resultsPath).toBe(
join(saveDir, "results.bqrs"),
);
expect(info.queryEvalInfo.resultsPaths.interpretedResultsPath).toBe(
expect(queryEvalInfo.resultsPaths.interpretedResultsPath).toBe(
join(saveDir, "interpretedResults.sarif"),
);
expect(info.dbItemPath).toBe(Uri.file("/abc").fsPath);
expect(queryEvalInfo.dbItemPath).toBe(Uri.file("/abc").fsPath);
});
it("should check if interpreted results can be created", async () => {
const info = createMockQueryInfo(true);
const queryEvalInfo = createMockQueryEvaluationInfo(true);
// "1"
expect(info.queryEvalInfo.canHaveInterpretedResults()).toBe(true);
expect(queryEvalInfo.canHaveInterpretedResults()).toBe(true);
(info.queryEvalInfo as any).databaseHasMetadataFile = false;
(queryEvalInfo as any).databaseHasMetadataFile = false;
// "2"
expect(info.queryEvalInfo.canHaveInterpretedResults()).toBe(false);
expect(queryEvalInfo.canHaveInterpretedResults()).toBe(false);
(info.queryEvalInfo as any).databaseHasMetadataFile = true;
info.metadata!.kind = undefined;
(queryEvalInfo as any).databaseHasMetadataFile = true;
queryEvalInfo.metadata!.kind = undefined;
// "3"
expect(info.queryEvalInfo.canHaveInterpretedResults()).toBe(false);
expect(queryEvalInfo.canHaveInterpretedResults()).toBe(false);
info.metadata!.kind = "table";
queryEvalInfo.metadata!.kind = "table";
// "4"
expect(info.queryEvalInfo.canHaveInterpretedResults()).toBe(false);
expect(queryEvalInfo.canHaveInterpretedResults()).toBe(false);
// Graphs are not interpreted unless canary is set
info.metadata!.kind = "graph";
queryEvalInfo.metadata!.kind = "graph";
// "5"
expect(info.queryEvalInfo.canHaveInterpretedResults()).toBe(false);
expect(queryEvalInfo.canHaveInterpretedResults()).toBe(false);
isCanarySpy.mockReturnValueOnce(true);
// "6"
expect(info.queryEvalInfo.canHaveInterpretedResults()).toBe(true);
expect(queryEvalInfo.canHaveInterpretedResults()).toBe(true);
});
[SELECT_QUERY_NAME, "other"].forEach((resultSetName) => {
@@ -102,11 +97,8 @@ describe("run-queries", () => {
},
],
});
const info = createMockQueryInfo();
const promise = info.queryEvalInfo.exportCsvResults(
cliServer,
csvLocation,
);
const queryEvalInfo = createMockQueryEvaluationInfo();
const promise = queryEvalInfo.exportCsvResults(cliServer, csvLocation);
const result = await promise;
expect(result).toBe(true);
@@ -145,8 +137,8 @@ describe("run-queries", () => {
},
],
});
const info = createMockQueryInfo();
const promise = info.queryEvalInfo.exportCsvResults(cliServer, csvLocation);
const queryEvalInfo = createMockQueryEvaluationInfo();
const promise = queryEvalInfo.exportCsvResults(cliServer, csvLocation);
const result = await promise;
expect(result).toBe(true);
@@ -169,125 +161,64 @@ describe("run-queries", () => {
const cliServer = createMockCliServer({
bqrsInfo: [{ "result-sets": [] }],
});
const info = createMockQueryInfo();
const result = await info.queryEvalInfo.exportCsvResults(
cliServer,
csvLocation,
);
const queryEvalInfo = createMockQueryEvaluationInfo();
const result = await queryEvalInfo.exportCsvResults(cliServer, csvLocation);
expect(result).toBe(false);
});
describe("compile", () => {
it("should compile", async () => {
const info = createMockQueryInfo();
const qs = createMockQueryServerClient();
const mockProgress = "progress-monitor";
const mockCancel = "cancel-token";
const mockQlProgram = {
dbschemePath: "",
libraryPath: [],
queryPath: "",
};
const results = await compileQueryLegacy(
qs as any,
mockQlProgram,
undefined,
info.queryEvalInfo,
mockProgress as any,
mockCancel as any,
qs.logger,
);
expect(results).toEqual([{ message: "err", severity: Severity.ERROR }]);
expect(qs.sendRequest).toHaveBeenCalledTimes(1);
expect(qs.sendRequest).toHaveBeenCalledWith(
compileQuery,
{
compilationOptions: {
computeNoLocationUrls: true,
failOnWarnings: false,
fastCompilation: false,
includeDilInQlo: true,
localChecking: false,
noComputeGetUrl: false,
noComputeToString: false,
computeDefaultStrings: true,
emitDebugInfo: true,
},
extraOptions: {
timeoutSecs: 5,
},
queryToCheck: mockQlProgram,
resultPath: info.queryEvalInfo.compileQueryPath,
target: { query: {} },
},
mockCancel,
mockProgress,
);
});
});
describe("register", () => {
it("should register", async () => {
const qs = createMockQueryServerClient();
const runner = new LegacyQueryRunner(qs);
const runner = new NewQueryRunner(qs);
const databaseUri = Uri.file("database-uri");
const datasetUri = Uri.file("dataset-uri");
const dbItem: DatabaseItem = {
const dbItem = mockDatabaseItem({
databaseUri,
contents: {
datasetUri,
},
} as any;
});
await runner.registerDatabase(dbItem);
expect(qs.sendRequest).toHaveBeenCalledTimes(1);
expect(qs.sendRequest).toHaveBeenCalledWith(registerDatabases, {
databases: [
{
dbDir: datasetUri.fsPath,
workingSet: "default",
},
],
databases: [databaseUri.fsPath],
});
});
it("should deregister", async () => {
const qs = createMockQueryServerClient();
const runner = new LegacyQueryRunner(qs);
const runner = new NewQueryRunner(qs);
const databaseUri = Uri.file("database-uri");
const datasetUri = Uri.file("dataset-uri");
const dbItem: DatabaseItem = {
const dbItem = mockDatabaseItem({
databaseUri,
contents: {
datasetUri,
},
} as any;
});
await runner.deregisterDatabase(dbItem);
expect(qs.sendRequest).toHaveBeenCalledTimes(1);
expect(qs.sendRequest).toHaveBeenCalledWith(deregisterDatabases, {
databases: [
{
dbDir: datasetUri.fsPath,
workingSet: "default",
},
],
databases: [databaseUri.fsPath],
});
});
});
let queryNum = 0;
function createMockQueryInfo(
function createMockQueryEvaluationInfo(
databaseHasMetadataFile = true,
saveDir = `save-dir${queryNum++}`,
) {
return new QueryInProgress(
return new QueryEvaluationInfo(
saveDir,
Uri.parse("file:///abc").fsPath,
databaseHasMetadataFile,
"my-scheme", // queryDbscheme,
undefined,
{
kind: "problem",
@@ -302,12 +233,7 @@ describe("run-queries", () => {
config: {
timeoutSecs: 5,
},
sendRequest: jest.fn().mockResolvedValue({
messages: [
{ message: "err", severity: Severity.ERROR },
{ message: "warn", severity: Severity.WARNING },
],
}),
sendRequest: jest.fn().mockResolvedValue({}),
logger: {
log: jest.fn(),
},