mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Refactor query evaluation code to separate UI concerns

Browse Source
This commit is contained in:
Dave Bartolomeo
2023-03-24 17:27:53 -04:00
parent 1e4672bb4c
commit f99f465365
16 changed files with 821 additions and 608 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
extensions/ql-vscode/src/common/logging/logger.ts
View File

@@ -3,7 +3,8 @@ export interface LogOptions {
trailingNewline?: boolean;
}
export interface Logger {
/** Minimal logger interface. */
export interface BaseLogger {
/**
* Writes the given log message, optionally followed by a newline.
* This function is asynchronous and will only resolve once the message is written
@@ -15,7 +16,10 @@ export interface Logger {
* @param options Optional settings.
*/
log(message: string, options?: LogOptions): Promise<void>;
}
/** Full logger interface, including a function to show the log in the UI. */
export interface Logger extends BaseLogger {
/**
* Reveal the logger channel in the UI.
*

6
extensions/ql-vscode/src/contextual/astBuilder.ts
View File

@@ -4,7 +4,7 @@ import { DatabaseItem } from "../local-databases";
import { ChildAstItem, AstItem } from "../astViewer";
import fileRangeFromURI from "./fileRangeFromURI";
import { Uri } from "vscode";
import { QueryWithResults } from "../run-queries-shared";
import { QueryOutputDir } from "../run-queries-shared";
/**
* A class that wraps a tree of QL results from a query that
@@ -14,12 +14,12 @@ export default class AstBuilder {
private roots: AstItem[] | undefined;
private bqrsPath: string;
constructor(
queryResults: QueryWithResults,
outputDir: QueryOutputDir,
private cli: CodeQLCliServer,
public db: DatabaseItem,
public fileName: Uri,
) {
this.bqrsPath = queryResults.query.resultsPaths.resultsPath;
this.bqrsPath = outputDir.bqrsPath;
}
async getRoots(): Promise<AstItem[]> {

13
extensions/ql-vscode/src/contextual/locationFinder.ts
View File

@@ -19,8 +19,9 @@ import {
runContextualQuery,
} from "./queryResolver";
import { CancellationToken, LocationLink, Uri } from "vscode";
import { QueryWithResults } from "../run-queries-shared";
import { QueryOutputDir } from "../run-queries-shared";
import { QueryRunner } from "../queryRunner";
import { QueryResultType } from "../pure/new-messages";
export const SELECT_QUERY_NAME = "#select";
export const TEMPLATE_NAME = "selectedSourceFile";
@@ -78,21 +79,23 @@ export async function getLocationsForUriString(
token,
templates,
);
if (results.successful) {
links.push(...(await getLinksFromResults(results, cli, db, filter)));
if (results.resultType === QueryResultType.SUCCESS) {
links.push(
...(await getLinksFromResults(results.outputDir, cli, db, filter)),
);
}
}
return links;
}
async function getLinksFromResults(
results: QueryWithResults,
outputDir: QueryOutputDir,
cli: CodeQLCliServer,
db: DatabaseItem,
filter: (srcFile: string, destFile: string) => boolean,
): Promise<FullLocationLink[]> {
const localLinks: FullLocationLink[] = [];
const bqrsPath = results.query.resultsPaths.resultsPath;
const bqrsPath = outputDir.bqrsPath;
const info = await cli.bqrsInfo(bqrsPath);
const selectInfo = getResultSetSchema(SELECT_QUERY_NAME, info);
if (isValidSelect(selectInfo)) {

33
extensions/ql-vscode/src/contextual/queryResolver.ts
View File

@@ -13,11 +13,10 @@ import {
import { KeyType, kindOfKeyType, nameOfKeyType, tagOfKeyType } from "./keyType";
import { CodeQLCliServer } from "../cli";
import { DatabaseItem } from "../local-databases";
import { extLogger } from "../common";
import { createInitialQueryInfo } from "../run-queries-shared";
import { CancellationToken, Uri } from "vscode";
import { extLogger, TeeLogger } from "../common";
import { CancellationToken } from "vscode";
import { ProgressCallback } from "../progress";
import { QueryRunner } from "../queryRunner";
import { CoreCompletedQuery, QueryRunner } from "../queryRunner";
import { redactableError } from "../pure/errors";
import { QLPACK_FILENAMES } from "../pure/ql";
@@ -169,32 +168,30 @@ export async function runContextualQuery(
progress: ProgressCallback,
token: CancellationToken,
templates: Record<string, string>,
) {
): Promise<CoreCompletedQuery> {
const { packPath, createdTempLockFile } = await resolveContextualQuery(
cli,
query,
);
const initialInfo = await createInitialQueryInfo(
Uri.file(query),
{
name: db.name,
databaseUri: db.databaseUri.toString(),
},
const queryRun = qs.createQueryRun(
db.databaseUri.fsPath,
{ queryPath: query, quickEvalPosition: undefined },
false,
getOnDiskWorkspaceFolders(),
queryStorageDir,
undefined,
templates,
);
void extLogger.log(
`Running contextual query ${query}; results will be stored in ${queryStorageDir}`,
`Running contextual query ${query}; results will be stored in ${queryRun.outputDir}`,
);
const queryResult = await qs.compileAndRunQueryAgainstDatabase(
db,
initialInfo,
queryStorageDir,
const results = await queryRun.evaluate(
progress,
token,
templates,
new TeeLogger(qs.logger, queryRun.outputDir.logPath),
);
if (createdTempLockFile) {
await removeTemporaryLockFile(packPath);
}
return queryResult;
return results;
}

29
extensions/ql-vscode/src/contextual/templateProvider.ts
View File

@@ -32,8 +32,7 @@ import {
runContextualQuery,
} from "./queryResolver";
import { isCanary, NO_CACHE_AST_VIEWER } from "../config";
import { QueryWithResults } from "../run-queries-shared";
import { QueryRunner } from "../queryRunner";
import { CoreCompletedQuery, QueryRunner } from "../queryRunner";
/**
* Runs templated CodeQL queries to find definitions in
@@ -155,17 +154,12 @@ export class TemplateQueryReferenceProvider implements ReferenceProvider {
}
}
type QueryWithDb = {
query: QueryWithResults;
dbUri: Uri;
};
/**
* Run templated CodeQL queries to produce AST information for
* source-language files.
*/
export class TemplatePrintAstProvider {
private cache: CachedOperation<QueryWithDb>;
private cache: CachedOperation<CoreCompletedQuery>;
constructor(
private cli: CodeQLCliServer,
@@ -173,7 +167,9 @@ export class TemplatePrintAstProvider {
private dbm: DatabaseManager,
private queryStorageDir: string,
) {
this.cache = new CachedOperation<QueryWithDb>(this.getAst.bind(this));
this.cache = new CachedOperation<CoreCompletedQuery>(
this.getAst.bind(this),
);
}
async provideAst(
@@ -186,14 +182,14 @@ export class TemplatePrintAstProvider {
"Cannot view the AST. Please select a valid source file inside a CodeQL database.",
);
}
const { query, dbUri } = this.shouldCache()
const completedQuery = this.shouldCache()
? await this.cache.get(fileUri.toString(), progress, token)
: await this.getAst(fileUri.toString(), progress, token);
return new AstBuilder(
query,
completedQuery.outputDir,
this.cli,
this.dbm.findDatabaseItem(dbUri)!,
this.dbm.findDatabaseItem(Uri.file(completedQuery.dbPath))!,
fileUri,
);
}
@@ -206,7 +202,7 @@ export class TemplatePrintAstProvider {
uriString: string,
progress: ProgressCallback,
token: CancellationToken,
): Promise<QueryWithDb> {
): Promise<CoreCompletedQuery> {
const uri = Uri.parse(uriString, true);
if (uri.scheme !== zipArchiveScheme) {
throw new Error(
@@ -242,7 +238,7 @@ export class TemplatePrintAstProvider {
[TEMPLATE_NAME]: zippedArchive.pathWithinSourceArchive,
};
const queryResult = await runContextualQuery(
const results = await runContextualQuery(
query,
db,
this.queryStorageDir,
@@ -252,10 +248,7 @@ export class TemplatePrintAstProvider {
token,
templates,
);
return {
query: queryResult,
dbUri: db.databaseUri,
};
return results;
}
}

2
extensions/ql-vscode/src/helpers.ts
View File

@@ -754,7 +754,7 @@ export async function tryGetQueryMetadata(
* Creates a file in the query directory that indicates when this query was created.
* This is important for keeping track of when queries should be removed.
*
* @param queryPath The directory that will containt all files relevant to a query result.
* @param queryPath The directory that will contain all files relevant to a query result.
* It does not need to exist.
*/
export async function createTimestampFile(storagePath: string) {

49
extensions/ql-vscode/src/legacy-query-server/legacyRunner.ts
View File

@@ -1,18 +1,19 @@
import { CancellationToken } from "vscode";
import { CodeQLCliServer } from "../cli";
import { ProgressCallback } from "../progress";
import { Logger } from "../common";
import { DatabaseItem } from "../local-databases";
import {
Dataset,
deregisterDatabases,
registerDatabases,
} from "../pure/legacy-messages";
import { InitialQueryInfo, LocalQueryInfo } from "../query-results";
import { QueryRunner } from "../queryRunner";
import { QueryWithResults } from "../run-queries-shared";
import { CoreQueryResults, CoreQueryTarget, QueryRunner } from "../queryRunner";
import { QueryOutputDir } from "../run-queries-shared";
import { QueryServerClient } from "./queryserver-client";
import {
clearCacheInDatabase,
compileAndRunQueryAgainstDatabase,
compileAndRunQueryAgainstDatabaseCore,
} from "./run-queries";
import { upgradeDatabaseExplicit } from "./upgrades";
@@ -21,10 +22,18 @@ export class LegacyQueryRunner extends QueryRunner {
super();
}
get cliServer() {
get cliServer(): CodeQLCliServer {
return this.qs.cliServer;
}
get customLogDirectory(): string | undefined {
return undefined;
}
get logger(): Logger {
return this.qs.logger;
}
async restartQueryServer(
progress: ProgressCallback,
token: CancellationToken,
@@ -47,25 +56,29 @@ export class LegacyQueryRunner extends QueryRunner {
): Promise<void> {
await clearCacheInDatabase(this.qs, dbItem, progress, token);
}
async compileAndRunQueryAgainstDatabase(
dbItem: DatabaseItem,
initialInfo: InitialQueryInfo,
queryStorageDir: string,
protected async compileAndRunQueryAgainstDatabaseCore(
dbPath: string,
query: CoreQueryTarget,
additionalPacks: string[],
generateEvalLog: boolean,
outputDir: QueryOutputDir,
progress: ProgressCallback,
token: CancellationToken,
templates?: Record<string, string>,
queryInfo?: LocalQueryInfo,
): Promise<QueryWithResults> {
return await compileAndRunQueryAgainstDatabase(
this.qs.cliServer,
templates: Record<string, string> | undefined,
logger: Logger,
): Promise<CoreQueryResults> {
return await compileAndRunQueryAgainstDatabaseCore(
this.qs,
dbItem,
initialInfo,
queryStorageDir,
dbPath,
query,
generateEvalLog,
additionalPacks,
outputDir,
progress,
token,
templates,
queryInfo,
logger,
);
}

523
extensions/ql-vscode/src/legacy-query-server/run-queries.ts
View File

@@ -1,28 +1,173 @@
import * as tmp from "tmp-promise";
import { basename, join } from "path";
import { basename } from "path";
import { CancellationToken, Uri } from "vscode";
import { LSPErrorCodes, ResponseError } from "vscode-languageclient";
import * as cli from "../cli";
import { DatabaseItem } from "../local-databases";
import {
getOnDiskWorkspaceFolders,
showAndLogErrorMessage,
DatabaseContentsWithDbScheme,
DatabaseItem,
DatabaseResolver,
} from "../local-databases";
import {
showAndLogExceptionWithTelemetry,
showAndLogWarningMessage,
tryGetQueryMetadata,
upgradesTmpDir,
} from "../helpers";
import { ProgressCallback } from "../progress";
import { QueryMetadata } from "../pure/interface-types";
import { extLogger, Logger, TeeLogger } from "../common";
import { extLogger, Logger } from "../common";
import * as messages from "../pure/legacy-messages";
import { InitialQueryInfo, LocalQueryInfo } from "../query-results";
import * as newMessages from "../pure/new-messages";
import * as qsClient from "./queryserver-client";
import { asError, getErrorMessage } from "../pure/helpers-pure";
import { compileDatabaseUpgradeSequence } from "./upgrades";
import { QueryEvaluationInfo, QueryWithResults } from "../run-queries-shared";
import { QueryEvaluationInfo, QueryOutputDir } from "../run-queries-shared";
import { redactableError } from "../pure/errors";
import { CoreQueryResults, CoreQueryTarget } from "../queryRunner";
import { Position } from "../pure/messages-shared";
async function compileQuery(
qs: qsClient.QueryServerClient,
program: messages.QlProgram,
quickEvalPosition: Position | undefined,
outputDir: QueryOutputDir,
progress: ProgressCallback,
token: CancellationToken,
logger: Logger,
): Promise<messages.CompilationMessage[]> {
let compiled: messages.CheckQueryResult | undefined;
try {
const target: messages.CompilationTarget = quickEvalPosition
? {
quickEval: { quickEvalPos: quickEvalPosition },
}
: { query: {} };
const params: messages.CompileQueryParams = {
compilationOptions: {
computeNoLocationUrls: true,
failOnWarnings: false,
fastCompilation: false,
includeDilInQlo: true,
localChecking: false,
noComputeGetUrl: false,
noComputeToString: false,
computeDefaultStrings: true,
emitDebugInfo: true,
},
extraOptions: {
timeoutSecs: qs.config.timeoutSecs,
},
queryToCheck: program,
resultPath: outputDir.compileQueryPath,
target,
};
// Update the active query logger every time there is a new request to compile.
// This isn't ideal because in situations where there are queries running
// in parallel, each query's log messages are interleaved. Fixing this
// properly will require a change in the query server.
qs.activeQueryLogger = logger;
compiled = await qs.sendRequest(
messages.compileQuery,
params,
token,
progress,
);
} finally {
void logger.log(" - - - COMPILATION DONE - - - ");
}
return (compiled?.messages || []).filter(
(msg) => msg.severity === messages.Severity.ERROR,
);
}
async function runQuery(
qs: qsClient.QueryServerClient,
upgradeQlo: string | undefined,
availableMlModels: cli.MlModelInfo[],
dbContents: DatabaseContentsWithDbScheme,
templates: Record<string, string> | undefined,
generateEvalLog: boolean,
outputDir: QueryOutputDir,
progress: ProgressCallback,
token: CancellationToken,
): Promise<messages.EvaluationResult> {
let result: messages.EvaluationResult | null = null;
const logPath = outputDir.logPath;
const callbackId = qs.registerCallback((res) => {
result = {
...res,
logFileLocation: logPath,
};
});
const availableMlModelUris: messages.MlModel[] = availableMlModels.map(
(model) => ({ uri: Uri.file(model.path).toString(true) }),
);
const queryToRun: messages.QueryToRun = {
resultsPath: outputDir.bqrsPath,
qlo: Uri.file(outputDir.compileQueryPath).toString(),
compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(),
allowUnknownTemplates: true,
templateValues: createSimpleTemplates(templates),
availableMlModels: availableMlModelUris,
id: callbackId,
timeoutSecs: qs.config.timeoutSecs,
};
const dataset: messages.Dataset = {
dbDir: dbContents.datasetUri.fsPath,
workingSet: "default",
};
if (
generateEvalLog &&
(await qs.cliServer.cliConstraints.supportsPerQueryEvalLog())
) {
await qs.sendRequest(messages.startLog, {
db: dataset,
logPath: outputDir.evalLogPath,
});
}
const params: messages.EvaluateQueriesParams = {
db: dataset,
evaluateId: callbackId,
queries: [queryToRun],
stopOnError: false,
useSequenceHint: false,
};
try {
await qs.sendRequest(messages.runQueries, params, token, progress);
if (qs.config.customLogDirectory) {
void showAndLogWarningMessage(
`Custom log directories are no longer supported. The "codeQL.runningQueries.customLogDirectory" setting is deprecated. Unset the setting to stop seeing this message. Query logs saved to ${logPath}.`,
);
}
} finally {
qs.unRegisterCallback(callbackId);
if (
generateEvalLog &&
(await qs.cliServer.cliConstraints.supportsPerQueryEvalLog())
) {
await qs.sendRequest(messages.endLog, {
db: dataset,
logPath: outputDir.evalLogPath,
});
}
}
return (
result || {
evaluationTime: 0,
message: "No result from server",
queryId: -1,
runId: callbackId,
resultType: messages.QueryResultType.OTHER_ERROR,
}
);
}
/**
* A collection of evaluation-time information about a query,
@@ -54,162 +199,6 @@ export class QueryInProgress {
);
/**/
}
get compiledQueryPath() {
return this.queryEvalInfo.compileQueryPath;
}
async run(
qs: qsClient.QueryServerClient,
upgradeQlo: string | undefined,
availableMlModels: cli.MlModelInfo[],
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
logger: Logger,
queryInfo: LocalQueryInfo | undefined,
): Promise<messages.EvaluationResult> {
if (!dbItem.contents || dbItem.error) {
throw new Error("Can't run query on invalid database.");
}
let result: messages.EvaluationResult | null = null;
const callbackId = qs.registerCallback((res) => {
result = {
...res,
logFileLocation: this.queryEvalInfo.logPath,
};
});
const availableMlModelUris: messages.MlModel[] = availableMlModels.map(
(model) => ({ uri: Uri.file(model.path).toString(true) }),
);
const queryToRun: messages.QueryToRun = {
resultsPath: this.queryEvalInfo.resultsPaths.resultsPath,
qlo: Uri.file(this.compiledQueryPath).toString(),
compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(),
allowUnknownTemplates: true,
templateValues: createSimpleTemplates(this.templates),
availableMlModels: availableMlModelUris,
id: callbackId,
timeoutSecs: qs.config.timeoutSecs,
};
const dataset: messages.Dataset = {
dbDir: dbItem.contents.datasetUri.fsPath,
workingSet: "default",
};
if (
queryInfo &&
(await qs.cliServer.cliConstraints.supportsPerQueryEvalLog())
) {
await qs.sendRequest(messages.startLog, {
db: dataset,
logPath: this.queryEvalInfo.evalLogPath,
});
}
const params: messages.EvaluateQueriesParams = {
db: dataset,
evaluateId: callbackId,
queries: [queryToRun],
stopOnError: false,
useSequenceHint: false,
};
try {
await qs.sendRequest(messages.runQueries, params, token, progress);
if (qs.config.customLogDirectory) {
void showAndLogWarningMessage(
`Custom log directories are no longer supported. The "codeQL.runningQueries.customLogDirectory" setting is deprecated. Unset the setting to stop seeing this message. Query logs saved to ${this.queryEvalInfo.logPath}.`,
);
}
} finally {
qs.unRegisterCallback(callbackId);
if (
queryInfo &&
(await qs.cliServer.cliConstraints.supportsPerQueryEvalLog())
) {
await qs.sendRequest(messages.endLog, {
db: dataset,
logPath: this.queryEvalInfo.evalLogPath,
});
if (await this.queryEvalInfo.hasEvalLog()) {
await this.queryEvalInfo.addQueryLogs(
queryInfo,
qs.cliServer,
logger,
);
} else {
void showAndLogWarningMessage(
`Failed to write structured evaluator log to ${this.queryEvalInfo.evalLogPath}.`,
);
}
}
}
return (
result || {
evaluationTime: 0,
message: "No result from server",
queryId: -1,
runId: callbackId,
resultType: messages.QueryResultType.OTHER_ERROR,
}
);
}
async compile(
qs: qsClient.QueryServerClient,
program: messages.QlProgram,
progress: ProgressCallback,
token: CancellationToken,
logger: Logger,
): Promise<messages.CompilationMessage[]> {
let compiled: messages.CheckQueryResult | undefined;
try {
const target = this.quickEvalPosition
? {
quickEval: { quickEvalPos: this.quickEvalPosition },
}
: { query: {} };
const params: messages.CompileQueryParams = {
compilationOptions: {
computeNoLocationUrls: true,
failOnWarnings: false,
fastCompilation: false,
includeDilInQlo: true,
localChecking: false,
noComputeGetUrl: false,
noComputeToString: false,
computeDefaultStrings: true,
emitDebugInfo: true,
},
extraOptions: {
timeoutSecs: qs.config.timeoutSecs,
},
queryToCheck: program,
resultPath: this.compiledQueryPath,
target,
};
// Update the active query logger every time there is a new request to compile.
// This isn't ideal because in situations where there are queries running
// in parallel, each query's log messages are interleaved. Fixing this
// properly will require a change in the query server.
qs.activeQueryLogger = logger;
compiled = await qs.sendRequest(
messages.compileQuery,
params,
token,
progress,
);
} finally {
void logger.log(" - - - COMPILATION DONE - - - ");
}
return (compiled?.messages || []).filter(
(msg) => msg.severity === messages.Severity.ERROR,
);
}
}
export async function clearCacheInDatabase(
@@ -237,10 +226,10 @@ export async function clearCacheInDatabase(
function reportNoUpgradePath(
qlProgram: messages.QlProgram,
query: QueryInProgress,
queryDbscheme: string,
): void {
throw new Error(
`Query ${qlProgram.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.`,
`Query ${qlProgram.queryPath} expects database scheme ${queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.`,
);
}
@@ -250,32 +239,27 @@ function reportNoUpgradePath(
async function compileNonDestructiveUpgrade(
qs: qsClient.QueryServerClient,
upgradeTemp: tmp.DirectoryResult,
query: QueryInProgress,
queryDbscheme: string,
qlProgram: messages.QlProgram,
dbItem: DatabaseItem,
dbContents: DatabaseContentsWithDbScheme,
progress: ProgressCallback,
token: CancellationToken,
): Promise<string> {
if (!dbItem?.contents?.dbSchemeUri) {
throw new Error("Database is invalid, and cannot be upgraded.");
}
// Dependencies may exist outside of the workspace and they are always on the resolved search path.
const upgradesPath = qlProgram.libraryPath;
const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(
dbItem.contents.dbSchemeUri.fsPath,
dbContents.dbSchemeUri.fsPath,
upgradesPath,
true,
query.queryDbscheme,
queryDbscheme,
);
if (!matchesTarget) {
reportNoUpgradePath(qlProgram, query);
reportNoUpgradePath(qlProgram, queryDbscheme);
}
const result = await compileDatabaseUpgradeSequence(
qs,
dbItem,
scripts,
upgradeTemp,
progress,
@@ -286,34 +270,67 @@ async function compileNonDestructiveUpgrade(
throw new Error(error);
}
// We can upgrade to the actual target
qlProgram.dbschemePath = query.queryDbscheme;
qlProgram.dbschemePath = queryDbscheme;
// We are new enough that we will always support single file upgrades.
return result.compiledUpgrade;
}
export async function compileAndRunQueryAgainstDatabase(
cliServer: cli.CodeQLCliServer,
qs: qsClient.QueryServerClient,
dbItem: DatabaseItem,
initialInfo: InitialQueryInfo,
queryStorageDir: string,
progress: ProgressCallback,
token: CancellationToken,
templates?: Record<string, string>,
queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
): Promise<QueryWithResults> {
if (!dbItem.contents || !dbItem.contents.dbSchemeUri) {
throw new Error(
`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`,
);
function translateLegacyResult(
legacyResult: messages.EvaluationResult,
): Omit<CoreQueryResults, "dispose"> {
let newResultType: newMessages.QueryResultType;
let newMessage = legacyResult.message;
switch (legacyResult.resultType) {
case messages.QueryResultType.SUCCESS:
newResultType = newMessages.QueryResultType.SUCCESS;
break;
case messages.QueryResultType.CANCELLATION:
newResultType = newMessages.QueryResultType.CANCELLATION;
break;
case messages.QueryResultType.OOM:
newResultType = newMessages.QueryResultType.OOM;
break;
case messages.QueryResultType.TIMEOUT:
// This is the only legacy result type that doesn't exist for the new query server. Format the
// messasge here, and let the later code treat is as `OTHER_ERROR`.
newResultType = newMessages.QueryResultType.OTHER_ERROR;
newMessage = `timed out after ${Math.round(
legacyResult.evaluationTime / 1000,
)} seconds`;
break;
case messages.QueryResultType.OTHER_ERROR:
default:
newResultType = newMessages.QueryResultType.OTHER_ERROR;
break;
}
// Get the workspace folder paths.
const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
return {
resultType: newResultType,
message: newMessage,
evaluationTime: legacyResult.evaluationTime,
};
}
export async function compileAndRunQueryAgainstDatabaseCore(
qs: qsClient.QueryServerClient,
dbPath: string,
query: CoreQueryTarget,
generateEvalLog: boolean,
additionalPacks: string[],
outputDir: QueryOutputDir,
progress: ProgressCallback,
token: CancellationToken,
templates: Record<string, string> | undefined,
logger: Logger,
): Promise<CoreQueryResults> {
const dbContents = await DatabaseResolver.resolveDatabaseContents(
Uri.file(dbPath),
);
// Figure out the library path for the query.
const packConfig = await cliServer.resolveLibraryPath(
diskWorkspaceFolders,
initialInfo.queryPath,
const packConfig = await qs.cliServer.resolveLibraryPath(
additionalPacks,
query.queryPath,
);
if (!packConfig.dbscheme) {
@@ -327,17 +344,15 @@ export async function compileAndRunQueryAgainstDatabase(
// won't trigger this check)
// This test will produce confusing results if we ever change the name of the database schema files.
const querySchemaName = basename(packConfig.dbscheme);
const dbSchemaName = basename(dbItem.contents.dbSchemeUri.fsPath);
const dbSchemaName = basename(dbContents.dbSchemeUri?.fsPath);
if (querySchemaName !== dbSchemaName) {
void extLogger.log(
`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`,
);
throw new Error(
`The query ${basename(
initialInfo.queryPath,
)} cannot be run against the selected database (${
dbItem.name
}): their target languages are different. Please select a different database and try again.`,
query.queryPath,
)} cannot be run against the selected database: their target languages are different. Please select a different database and try again.`,
);
}
@@ -349,20 +364,14 @@ export async function compileAndRunQueryAgainstDatabase(
// Since we are compiling and running a query against a database,
// we use the database's DB scheme here instead of the DB scheme
// from the current document's project.
dbschemePath: dbItem.contents.dbSchemeUri.fsPath,
queryPath: initialInfo.queryPath,
dbschemePath: dbContents.dbSchemeUri.fsPath,
queryPath: query.queryPath,
};
// Read the query metadata if possible, to use in the UI.
const metadata = await tryGetQueryMetadata(cliServer, qlProgram.queryPath);
let availableMlModels: cli.MlModelInfo[] = [];
try {
availableMlModels = (
await cliServer.resolveMlModels(
diskWorkspaceFolders,
initialInfo.queryPath,
)
await qs.cliServer.resolveMlModels(additionalPacks, query.queryPath)
).models;
if (availableMlModels.length) {
void extLogger.log(
@@ -381,57 +390,53 @@ export async function compileAndRunQueryAgainstDatabase(
);
}
const hasMetadataFile = await dbItem.hasMetadataFile();
const query = new QueryInProgress(
join(queryStorageDir, initialInfo.id),
dbItem.databaseUri.fsPath,
hasMetadataFile,
packConfig.dbscheme,
initialInfo.quickEvalPosition,
metadata,
templates,
);
const logger = new TeeLogger(qs.logger, query.queryEvalInfo.logPath);
await query.queryEvalInfo.createTimestampFile();
let upgradeDir: tmp.DirectoryResult | undefined;
try {
upgradeDir = await tmp.dir({ dir: upgradesTmpDir, unsafeCleanup: true });
const upgradeQlo = await compileNonDestructiveUpgrade(
qs,
upgradeDir,
query,
packConfig.dbscheme,
qlProgram,
dbItem,
dbContents,
progress,
token,
);
let errors;
try {
errors = await query.compile(qs, qlProgram, progress, token, logger);
errors = await compileQuery(
qs,
qlProgram,
query.quickEvalPosition,
outputDir,
progress,
token,
logger,
);
} catch (e) {
if (
e instanceof ResponseError &&
e.code === LSPErrorCodes.RequestCancelled
) {
return createSyntheticResult(query, "Query cancelled");
return createSyntheticResult("Query cancelled");
} else {
throw e;
}
}
if (errors.length === 0) {
const result = await query.run(
const result = await runQuery(
qs,
upgradeQlo,
availableMlModels,
dbItem,
dbContents,
templates,
generateEvalLog,
outputDir,
progress,
token,
logger,
queryInfo,
);
if (result.resultType !== messages.QueryResultType.SUCCESS) {
const error = result.message
? redactableError`${result.message}`
@@ -439,14 +444,9 @@ export async function compileAndRunQueryAgainstDatabase(
void extLogger.log(error.fullMessage);
void showAndLogExceptionWithTelemetry(error);
}
const message = formatLegacyMessage(result);
return {
query: query.queryEvalInfo,
message,
result,
successful: result.resultType === messages.QueryResultType.SUCCESS,
logFileLocation: result.logFileLocation,
...translateLegacyResult(result),
};
} else {
// Error dialogs are limited in size and scrollability,
@@ -454,7 +454,7 @@ export async function compileAndRunQueryAgainstDatabase(
// and direct the user to the output window for the detailed compilation messages.
// However we don't show quick eval errors there so we need to display them anyway.
void logger.log(
`Failed to compile query ${initialInfo.queryPath} against database scheme ${qlProgram.dbschemePath}:`,
`Failed to compile query ${query.queryPath} against database scheme ${qlProgram.dbschemePath}:`,
);
const formattedMessages: string[] = [];
@@ -465,22 +465,12 @@ export async function compileAndRunQueryAgainstDatabase(
formattedMessages.push(formatted);
void logger.log(formatted);
}
if (initialInfo.isQuickEval && formattedMessages.length <= 2) {
// If there are more than 2 error messages, they will not be displayed well in a popup
// and will be trimmed by the function displaying the error popup. Accordingly, we only
// try to show the errors if there are 2 or less, otherwise we direct the user to the log.
void showAndLogErrorMessage(
`Quick evaluation compilation failed: ${formattedMessages.join(
"\n",
)}`,
);
} else {
void showAndLogErrorMessage(
(initialInfo.isQuickEval ? "Quick evaluation" : "Query") +
compilationFailedErrorTail,
);
}
return createSyntheticResult(query, "Query had compilation errors");
return {
evaluationTime: 0,
resultType: newMessages.QueryResultType.COMPILATION_ERROR,
message: formattedMessages[0],
};
}
} finally {
try {
@@ -493,11 +483,6 @@ export async function compileAndRunQueryAgainstDatabase(
}
}
const compilationFailedErrorTail =
" compilation failed. Please make sure there are no errors in the query, the database is up to date," +
" and the query and database use the same target language. For more details on the error, go to View > Output," +
" and choose CodeQL Query Server from the dropdown.";
export function formatLegacyMessage(result: messages.EvaluationResult) {
switch (result.resultType) {
case messages.QueryResultType.CANCELLATION:
@@ -521,21 +506,11 @@ export function formatLegacyMessage(result: messages.EvaluationResult) {
/**
* Create a synthetic result for a query that failed to compile.
*/
function createSyntheticResult(
query: QueryInProgress,
message: string,
): QueryWithResults {
function createSyntheticResult(message: string): CoreQueryResults {
return {
query: query.queryEvalInfo,
evaluationTime: 0,
resultType: newMessages.QueryResultType.OTHER_ERROR,
message,
result: {
evaluationTime: 0,
queryId: 0,
resultType: messages.QueryResultType.OTHER_ERROR,
message,
runId: 0,
},
successful: false,
};
}

7
extensions/ql-vscode/src/legacy-query-server/upgrades.ts
View File

@@ -27,18 +27,11 @@ const MAX_UPGRADE_MESSAGE_LINES = 10;
*/
export async function compileDatabaseUpgradeSequence(
qs: qsClient.QueryServerClient,
dbItem: DatabaseItem,
resolvedSequence: string[],
currentUpgradeTmp: tmp.DirectoryResult,
progress: ProgressCallback,
token: vscode.CancellationToken,
): Promise<messages.CompileUpgradeSequenceResult> {
if (
dbItem.contents === undefined ||
dbItem.contents.dbSchemeUri === undefined
) {
throw new Error("Database is invalid, and cannot be upgraded.");
}
// If possible just compile the upgrade sequence
return await qs.sendRequest(
messages.compileUpgradeSequence,

13
extensions/ql-vscode/src/local-databases.ts
View File

@@ -95,6 +95,10 @@ export interface DatabaseContents {
dbSchemeUri?: vscode.Uri;
}
export interface DatabaseContentsWithDbScheme extends DatabaseContents {
dbSchemeUri: vscode.Uri; // Always present
}
/**
* An error thrown when we cannot find a valid database in a putative
* database directory.
@@ -165,7 +169,7 @@ async function getDbSchemeFiles(dbDirectory: string): Promise<string[]> {
export class DatabaseResolver {
public static async resolveDatabaseContents(
uri: vscode.Uri,
): Promise<DatabaseContents> {
): Promise<DatabaseContentsWithDbScheme> {
if (uri.scheme !== "file") {
throw new Error(
`Database URI scheme '${uri.scheme}' not supported; only 'file' URIs are supported.`,
@@ -199,9 +203,12 @@ export class DatabaseResolver {
`Database '${databasePath}' contains multiple CodeQL dbschemes under '${dbPath}'.`,
);
} else {
contents.dbSchemeUri = vscode.Uri.file(resolve(dbPath, dbSchemeFiles[0]));
const dbSchemeUri = vscode.Uri.file(resolve(dbPath, dbSchemeFiles[0]));
return {
...contents,
dbSchemeUri,
};
}
return contents;
}
public static async resolveDatabase(

14
extensions/ql-vscode/src/query-results.ts
View File

@@ -16,7 +16,11 @@ import {
DatabaseInfo,
} from "./pure/interface-types";
import { QueryStatus } from "./query-status";
import { QueryEvaluationInfo, QueryWithResults } from "./run-queries-shared";
import {
EvaluatorLogPaths,
QueryEvaluationInfo,
QueryWithResults,
} from "./run-queries-shared";
import { formatLegacyMessage } from "./legacy-query-server/run-queries";
import { sarifParser } from "./sarif-parser";
@@ -253,6 +257,14 @@ export class LocalQueryInfo {
this.initialInfo.userSpecifiedLabel = label;
}
/** Sets the paths to the various structured evaluator logs. */
public setEvaluatorLogPaths(logPaths: EvaluatorLogPaths): void {
this.evalLogLocation = logPaths.log;
this.evalLogSummaryLocation = logPaths.humanReadableSummary;
this.jsonEvalLogSummaryLocation = logPaths.jsonSummary;
this.evalLogSummarySymbolsLocation = logPaths.summarySymbols;
}
/**
* The query's file name, unless it is a quick eval.
* Queries run through quick evaluation are not usually the entire query file.

50
extensions/ql-vscode/src/query-server/query-runner.ts
View File

@@ -9,22 +9,32 @@ import {
registerDatabases,
upgradeDatabase,
} from "../pure/new-messages";
import { InitialQueryInfo, LocalQueryInfo } from "../query-results";
import { QueryRunner } from "../queryRunner";
import { QueryWithResults } from "../run-queries-shared";
import { CoreQueryResults, CoreQueryTarget, QueryRunner } from "../queryRunner";
import { QueryServerClient } from "./queryserver-client";
import { compileAndRunQueryAgainstDatabase } from "./run-queries";
import { compileAndRunQueryAgainstDatabaseCore } from "./run-queries";
import * as vscode from "vscode";
import { getOnDiskWorkspaceFolders } from "../helpers";
import { CodeQLCliServer } from "../cli";
import { Logger } from "../common";
import { QueryOutputDir } from "../run-queries-shared";
export class NewQueryRunner extends QueryRunner {
constructor(public readonly qs: QueryServerClient) {
super();
}
get cliServer() {
get cliServer(): CodeQLCliServer {
return this.qs.cliServer;
}
get customLogDirectory(): string | undefined {
return this.qs.config.customLogDirectory;
}
get logger(): Logger {
return this.qs.logger;
}
async restartQueryServer(
progress: ProgressCallback,
token: CancellationToken,
@@ -57,25 +67,29 @@ export class NewQueryRunner extends QueryRunner {
};
await this.qs.sendRequest(clearCache, params, token, progress);
}
async compileAndRunQueryAgainstDatabase(
dbItem: DatabaseItem,
initialInfo: InitialQueryInfo,
queryStorageDir: string,
protected async compileAndRunQueryAgainstDatabaseCore(
dbPath: string,
query: CoreQueryTarget,
additionalPacks: string[],
generateEvalLog: boolean,
outputDir: QueryOutputDir,
progress: ProgressCallback,
token: CancellationToken,
templates?: Record<string, string>,
queryInfo?: LocalQueryInfo,
): Promise<QueryWithResults> {
return await compileAndRunQueryAgainstDatabase(
this.qs.cliServer,
templates: Record<string, string> | undefined,
logger: Logger,
): Promise<CoreQueryResults> {
return await compileAndRunQueryAgainstDatabaseCore(
this.qs,
dbItem,
initialInfo,
queryStorageDir,
dbPath,
query,
generateEvalLog,
additionalPacks,
outputDir,
progress,
token,
templates,
queryInfo,
logger,
);
}

175
extensions/ql-vscode/src/query-server/run-queries.ts
View File

@@ -1,21 +1,10 @@
import { join } from "path";
import { CancellationToken } from "vscode";
import * as cli from "../cli";
import { ProgressCallback } from "../progress";
import { DatabaseItem } from "../local-databases";
import {
getOnDiskWorkspaceFolders,
showAndLogExceptionWithTelemetry,
showAndLogWarningMessage,
tryGetQueryMetadata,
} from "../helpers";
import { extLogger, TeeLogger } from "../common";
import * as messages from "../pure/new-messages";
import { QueryResultType } from "../pure/legacy-messages";
import { InitialQueryInfo, LocalQueryInfo } from "../query-results";
import { QueryEvaluationInfo, QueryWithResults } from "../run-queries-shared";
import { QueryOutputDir } from "../run-queries-shared";
import * as qsClient from "./queryserver-client";
import { redactableError } from "../pure/errors";
import { CoreQueryResults, CoreQueryTarget } from "../queryRunner";
import { Logger } from "../common";
/**
* run-queries.ts
@@ -31,140 +20,58 @@ import { redactableError } from "../pure/errors";
* output and results.
*/
export async function compileAndRunQueryAgainstDatabase(
cliServer: cli.CodeQLCliServer,
export async function compileAndRunQueryAgainstDatabaseCore(
qs: qsClient.QueryServerClient,
dbItem: DatabaseItem,
initialInfo: InitialQueryInfo,
queryStorageDir: string,
dbPath: string,
query: CoreQueryTarget,
generateEvalLog: boolean,
additionalPacks: string[],
outputDir: QueryOutputDir,
progress: ProgressCallback,
token: CancellationToken,
templates?: Record<string, string>,
queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
): Promise<QueryWithResults> {
if (!dbItem.contents || !dbItem.contents.dbSchemeUri) {
throw new Error(
`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`,
);
}
templates: Record<string, string> | undefined,
logger: Logger,
): Promise<CoreQueryResults> {
const target =
query.quickEvalPosition !== undefined
? {
quickEval: { quickEvalPos: query.quickEvalPosition },
}
: { query: {} };
// Read the query metadata if possible, to use in the UI.
const metadata = await tryGetQueryMetadata(cliServer, initialInfo.queryPath);
const hasMetadataFile = await dbItem.hasMetadataFile();
const query = new QueryEvaluationInfo(
join(queryStorageDir, initialInfo.id),
dbItem.databaseUri.fsPath,
hasMetadataFile,
initialInfo.quickEvalPosition,
metadata,
);
if (!dbItem.contents || dbItem.error) {
throw new Error("Can't run query on invalid database.");
}
const target = query.quickEvalPosition
? {
quickEval: { quickEvalPos: query.quickEvalPosition },
}
: { query: {} };
const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
const extensionPacks = (await qs.cliServer.useExtensionPacks())
? Object.keys(await qs.cliServer.resolveQlpacks(diskWorkspaceFolders, true))
? Object.keys(await qs.cliServer.resolveQlpacks(additionalPacks, true))
: undefined;
const db = dbItem.databaseUri.fsPath;
const logPath = queryInfo ? query.evalLogPath : undefined;
const evalLogPath = generateEvalLog ? outputDir.evalLogPath : undefined;
const queryToRun: messages.RunQueryParams = {
db,
additionalPacks: diskWorkspaceFolders,
db: dbPath,
additionalPacks,
externalInputs: {},
singletonExternalInputs: templates || {},
outputPath: query.resultsPaths.resultsPath,
queryPath: initialInfo.queryPath,
dilPath: query.dilPath,
logPath,
outputPath: outputDir.bqrsPath,
queryPath: query.queryPath,
dilPath: outputDir.dilPath,
logPath: evalLogPath,
target,
extensionPacks,
};
const logger = new TeeLogger(qs.logger, query.logPath);
await query.createTimestampFile();
let result: messages.RunQueryResult | undefined;
try {
// Update the active query logger every time there is a new request to compile.
// This isn't ideal because in situations where there are queries running
// in parallel, each query's log messages are interleaved. Fixing this
// properly will require a change in the query server.
qs.activeQueryLogger = logger;
result = await qs.sendRequest(
messages.runQuery,
queryToRun,
token,
progress,
);
if (qs.config.customLogDirectory) {
void showAndLogWarningMessage(
`Custom log directories are no longer supported. The "codeQL.runningQueries.customLogDirectory" setting is deprecated. Unset the setting to stop seeing this message. Query logs saved to ${query.logPath}.`,
);
}
} finally {
if (queryInfo) {
if (await query.hasEvalLog()) {
await query.addQueryLogs(queryInfo, qs.cliServer, logger);
} else {
void showAndLogWarningMessage(
`Failed to write structured evaluator log to ${query.evalLogPath}.`,
);
}
}
}
if (result.resultType !== messages.QueryResultType.SUCCESS) {
const message = result?.message
? redactableError`${result.message}`
: redactableError`Failed to run query`;
void extLogger.log(message.fullMessage);
void showAndLogExceptionWithTelemetry(
redactableError`Failed to run query: ${message}`,
);
}
let message;
switch (result.resultType) {
case messages.QueryResultType.CANCELLATION:
message = `cancelled after ${Math.round(
result.evaluationTime / 1000,
)} seconds`;
break;
case messages.QueryResultType.OOM:
message = "out of memory";
break;
case messages.QueryResultType.SUCCESS:
message = `finished in ${Math.round(
result.evaluationTime / 1000,
)} seconds`;
break;
case messages.QueryResultType.COMPILATION_ERROR:
message = `compilation failed: ${result.message}`;
break;
case messages.QueryResultType.OTHER_ERROR:
default:
message = result.message ? `failed: ${result.message}` : "failed";
break;
}
const successful = result.resultType === messages.QueryResultType.SUCCESS;
// Update the active query logger every time there is a new request to compile.
// This isn't ideal because in situations where there are queries running
// in parallel, each query's log messages are interleaved. Fixing this
// properly will require a change in the query server.
qs.activeQueryLogger = logger;
const result = await qs.sendRequest(
messages.runQuery,
queryToRun,
token,
progress,
);
return {
query,
result: {
evaluationTime: result.evaluationTime,
queryId: 0,
resultType: successful
? QueryResultType.SUCCESS
: QueryResultType.OTHER_ERROR,
runId: 0,
message,
},
message,
successful,
resultType: result.resultType,
message: result.message,
evaluationTime: result.evaluationTime,
};
}

275
extensions/ql-vscode/src/queryRunner.ts
View File

@@ -3,7 +3,77 @@ import { CodeQLCliServer } from "./cli";
import { ProgressCallback } from "./progress";
import { DatabaseItem } from "./local-databases";
import { InitialQueryInfo, LocalQueryInfo } from "./query-results";
import { QueryWithResults } from "./run-queries-shared";
import {
EvaluatorLogPaths,
generateEvalLogSummaries,
logEndSummary,
QueryEvaluationInfo,
QueryOutputDir,
QueryWithResults,
} from "./run-queries-shared";
import { Position, QueryResultType } from "./pure/new-messages";
import {
createTimestampFile,
getOnDiskWorkspaceFolders,
showAndLogExceptionWithTelemetry,
showAndLogWarningMessage,
tryGetQueryMetadata,
} from "./helpers";
import { basename, join } from "path";
import { BaseLogger, extLogger, Logger, TeeLogger } from "./common";
import { redactableError } from "./pure/errors";
import { nanoid } from "nanoid";
export interface CoreQueryTarget {
/** The full path to the query. */
queryPath: string;
/**
* Optional position of text to be used as QuickEval target. This need not be in the same file as
* `query`.
*/
quickEvalPosition?: Position;
}
export interface CoreQueryResults {
readonly resultType: QueryResultType;
readonly message: string | undefined;
readonly evaluationTime: number;
}
export interface CoreQueryRun {
readonly queryTarget: CoreQueryTarget;
readonly dbPath: string;
readonly id: string;
readonly outputDir: QueryOutputDir;
evaluate(
progress: ProgressCallback,
token: CancellationToken,
logger: BaseLogger,
): Promise<CoreCompletedQuery>;
}
/** Includes both the results of the query and the initial information from `CoreQueryRun`. */
export type CoreCompletedQuery = CoreQueryResults &
Omit<CoreQueryRun, "evaluate">;
function formatResultMessage(result: CoreQueryResults): string {
switch (result.resultType) {
case QueryResultType.CANCELLATION:
return `cancelled after ${Math.round(
result.evaluationTime / 1000,
)} seconds`;
case QueryResultType.OOM:
return "out of memory";
case QueryResultType.SUCCESS:
return `finished in ${Math.round(result.evaluationTime / 1000)} seconds`;
case QueryResultType.COMPILATION_ERROR:
return `compilation failed: ${result.message}`;
case QueryResultType.OTHER_ERROR:
default:
return result.message ? `failed: ${result.message}` : "failed";
}
}
export abstract class QueryRunner {
abstract restartQueryServer(
@@ -12,6 +82,8 @@ export abstract class QueryRunner {
): Promise<void>;
abstract cliServer: CodeQLCliServer;
abstract customLogDirectory: string | undefined;
abstract logger: Logger;
abstract onStart(
arg0: (
@@ -25,15 +97,210 @@ export abstract class QueryRunner {
token: CancellationToken,
): Promise<void>;
abstract compileAndRunQueryAgainstDatabase(
dbItem: DatabaseItem,
public async compileAndRunQueryAgainstDatabase(
db: DatabaseItem,
initialInfo: InitialQueryInfo,
queryStorageDir: string,
progress: ProgressCallback,
token: CancellationToken,
templates?: Record<string, string>,
queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
): Promise<QueryWithResults>;
): Promise<QueryWithResults> {
const queryTarget: CoreQueryTarget = {
queryPath: initialInfo.queryPath,
quickEvalPosition: initialInfo.quickEvalPosition,
};
const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
const queryRun = this.createQueryRun(
db.databaseUri.fsPath,
queryTarget,
queryInfo !== undefined,
diskWorkspaceFolders,
queryStorageDir,
initialInfo.id,
templates,
);
await createTimestampFile(queryRun.outputDir.querySaveDir);
const logPath = queryRun.outputDir.logPath;
if (this.customLogDirectory) {
void showAndLogWarningMessage(
`Custom log directories are no longer supported. The "codeQL.runningQueries.customLogDirectory" setting is deprecated. Unset the setting to stop seeing this message. Query logs saved to ${logPath}`,
);
}
const logger = new TeeLogger(this.logger, logPath);
const coreResults = await queryRun.evaluate(progress, token, logger);
if (queryInfo !== undefined) {
const evalLogPaths = await this.summarizeEvalLog(
coreResults.resultType,
queryRun.outputDir,
logger,
);
if (evalLogPaths !== undefined) {
queryInfo.setEvaluatorLogPaths(evalLogPaths);
}
}
return await this.getCompletedQueryInfo(
db,
queryTarget,
queryRun.outputDir,
coreResults,
);
}
/**
* Generate summaries of the structured evaluator log.
*/
public async summarizeEvalLog(
resultType: QueryResultType,
outputDir: QueryOutputDir,
logger: BaseLogger,
): Promise<EvaluatorLogPaths | undefined> {
const evalLogPaths = await generateEvalLogSummaries(
this.cliServer,
outputDir,
);
if (evalLogPaths !== undefined) {
if (evalLogPaths.endSummary !== undefined) {
void logEndSummary(evalLogPaths.endSummary, logger); // Logged asynchrnously
}
} else {
// Raw evaluator log was not found. Notify the user, unless we know why it wasn't found.
switch (resultType) {
case QueryResultType.COMPILATION_ERROR:
case QueryResultType.DBSCHEME_MISMATCH_NAME:
case QueryResultType.DBSCHEME_NO_UPGRADE:
// In these cases, the evaluator was never invoked anyway, so don't bother warning.
break;
default:
void showAndLogWarningMessage(
`Failed to write structured evaluator log to ${outputDir.evalLogPath}.`,
);
break;
}
}
return evalLogPaths;
}
/**
* Gets a `QueryWithResults` containing information about the evaluation of the query and its
* result, in the form expected by the query history UI.
*/
public async getCompletedQueryInfo(
dbItem: DatabaseItem,
queryTarget: CoreQueryTarget,
outputDir: QueryOutputDir,
results: CoreQueryResults,
): Promise<QueryWithResults> {
// Read the query metadata if possible, to use in the UI.
const metadata = await tryGetQueryMetadata(
this.cliServer,
queryTarget.queryPath,
);
const query = new QueryEvaluationInfo(
outputDir.querySaveDir,
dbItem.databaseUri.fsPath,
await dbItem.hasMetadataFile(),
queryTarget.quickEvalPosition,
metadata,
);
if (results.resultType !== QueryResultType.SUCCESS) {
const message = results.message
? redactableError`${results.message}`
: redactableError`Failed to run query`;
void extLogger.log(message.fullMessage);
void showAndLogExceptionWithTelemetry(
redactableError`Failed to run query: ${message}`,
);
}
const message = formatResultMessage(results);
const successful = results.resultType === QueryResultType.SUCCESS;
return {
query,
result: {
evaluationTime: results.evaluationTime,
queryId: 0,
resultType: successful
? QueryResultType.SUCCESS
: QueryResultType.OTHER_ERROR,
runId: 0,
message,
},
message,
successful,
};
}
/**
* Create a `CoreQueryRun` object. This creates an object whose `evaluate()` function can be
* called to actually evaluate the query. The returned object also contains information about the
* query evaluation that is known even before evaluation starts, including the unique ID of the
* evaluation and the path to its output directory.
*/
public createQueryRun(
dbPath: string,
query: CoreQueryTarget,
generateEvalLog: boolean,
additionalPacks: string[],
queryStorageDir: string,
id: string | undefined,
templates: Record<string, string> | undefined,
): CoreQueryRun {
const actualId = id ?? `${basename(query.queryPath)}-${nanoid()}`;
const outputDir = new QueryOutputDir(join(queryStorageDir, actualId));
return {
queryTarget: query,
dbPath,
id: actualId,
outputDir,
evaluate: async (
progress: ProgressCallback,
token: CancellationToken,
logger: BaseLogger,
): Promise<CoreCompletedQuery> => {
return {
id: actualId,
outputDir,
dbPath,
queryTarget: query,
...(await this.compileAndRunQueryAgainstDatabaseCore(
dbPath,
query,
additionalPacks,
generateEvalLog,
outputDir,
progress,
token,
templates,
logger,
)),
};
},
};
}
/**
* Overridden in subclasses to evaluate the query via the query server and return the results.
*/
protected abstract compileAndRunQueryAgainstDatabaseCore(
dbPath: string,
query: CoreQueryTarget,
additionalPacks: string[],
generateEvalLog: boolean,
outputDir: QueryOutputDir,
progress: ProgressCallback,
token: CancellationToken,
templates: Record<string, string> | undefined,
logger: BaseLogger,
): Promise<CoreQueryResults>;
abstract deregisterDatabase(
progress: ProgressCallback,

225
extensions/ql-vscode/src/run-queries-shared.ts
View File

@@ -20,18 +20,14 @@ import {
remove,
readdir,
} from "fs-extra";
import {
ensureMetadataIsComplete,
InitialQueryInfo,
LocalQueryInfo,
} from "./query-results";
import { ensureMetadataIsComplete, InitialQueryInfo } from "./query-results";
import { isQuickQueryPath } from "./quick-query";
import { nanoid } from "nanoid";
import { CodeQLCliServer } from "./cli";
import { SELECT_QUERY_NAME } from "./contextual/locationFinder";
import { DatabaseManager } from "./local-databases";
import { DecodedBqrsChunk, EntityValue } from "./pure/bqrs-cli-types";
import { extLogger, Logger } from "./common";
import { BaseLogger, extLogger } from "./common";
import { generateSummarySymbolsFile } from "./log-insights/summary-parser";
import { getErrorMessage } from "./pure/helpers-pure";
@@ -42,7 +38,18 @@ import { getErrorMessage } from "./pure/helpers-pure";
* Compiling and running QL queries.
*/
export function findQueryLogFile(resultPath: string): string {
/**
* Holds the paths to the various structured log summary files generated for a query evaluation.
*/
export interface EvaluatorLogPaths {
log: string;
humanReadableSummary: string | undefined;
endSummary: string | undefined;
jsonSummary: string | undefined;
summarySymbols: string | undefined;
}
function findQueryLogFile(resultPath: string): string {
return join(resultPath, "query.log");
}
@@ -66,20 +73,11 @@ function findQueryEvalLogEndSummaryFile(resultPath: string): string {
return join(resultPath, "evaluator-log-end.summary");
}
export class QueryEvaluationInfo {
/**
* Note that in the {@link readQueryHistoryFromFile} method, we create a QueryEvaluationInfo instance
* by explicitly setting the prototype in order to avoid calling this constructor.
*/
constructor(
public readonly querySaveDir: string,
public readonly dbItemPath: string,
private readonly databaseHasMetadataFile: boolean,
public readonly quickEvalPosition?: messages.Position,
public readonly metadata?: QueryMetadata,
) {
/**/
}
/**
* Provides paths to the files that can be generated in the output directory for a query evaluation.
*/
export class QueryOutputDir {
constructor(public readonly querySaveDir: string) {}
get dilPath() {
return join(this.querySaveDir, "results.dil");
@@ -120,9 +118,29 @@ export class QueryEvaluationInfo {
return findQueryEvalLogEndSummaryFile(this.querySaveDir);
}
get bqrsPath() {
return join(this.querySaveDir, "results.bqrs");
}
}
export class QueryEvaluationInfo extends QueryOutputDir {
/**
* Note that in the {@link readQueryHistoryFromFile} method, we create a QueryEvaluationInfo instance
* by explicitly setting the prototype in order to avoid calling this constructor.
*/
constructor(
querySaveDir: string,
public readonly dbItemPath: string,
private readonly databaseHasMetadataFile: boolean,
public readonly quickEvalPosition?: messages.Position,
public readonly metadata?: QueryMetadata,
) {
super(querySaveDir);
}
get resultsPaths() {
return {
resultsPath: join(this.querySaveDir, "results.bqrs"),
resultsPath: this.bqrsPath,
interpretedResultsPath: join(
this.querySaveDir,
this.metadata?.kind === "graph"
@@ -228,85 +246,6 @@ export class QueryEvaluationInfo {
return pathExists(this.evalLogPath);
}
/**
* Add the structured evaluator log to the query evaluation info.
*/
async addQueryLogs(
queryInfo: LocalQueryInfo,
cliServer: CodeQLCliServer,
logger: Logger,
) {
queryInfo.evalLogLocation = this.evalLogPath;
queryInfo.evalLogSummaryLocation =
await this.generateHumanReadableLogSummary(cliServer);
void this.logEndSummary(queryInfo.evalLogSummaryLocation, logger); // Logged asynchrnously
if (isCanary()) {
// Generate JSON summary for viewer.
await cliServer.generateJsonLogSummary(
this.evalLogPath,
this.jsonEvalLogSummaryPath,
);
queryInfo.jsonEvalLogSummaryLocation = this.jsonEvalLogSummaryPath;
await generateSummarySymbolsFile(
this.evalLogSummaryPath,
this.evalLogSummarySymbolsPath,
);
queryInfo.evalLogSummarySymbolsLocation = this.evalLogSummarySymbolsPath;
}
}
/**
* Calls the appropriate CLI command to generate a human-readable log summary.
* @param qs The query server client.
* @returns The path to the log summary, or `undefined` if the summary could not be generated. */
private async generateHumanReadableLogSummary(
cliServer: CodeQLCliServer,
): Promise<string | undefined> {
try {
await cliServer.generateLogSummary(
this.evalLogPath,
this.evalLogSummaryPath,
this.evalLogEndSummaryPath,
);
return this.evalLogSummaryPath;
} catch (e) {
void showAndLogWarningMessage(
`Failed to generate human-readable structured evaluator log summary. Reason: ${getErrorMessage(
e,
)}`,
);
return undefined;
}
}
/**
* Logs the end summary to the Output window and log file.
* @param logSummaryPath Path to the human-readable log summary
* @param qs The query server client.
*/
private async logEndSummary(
logSummaryPath: string | undefined,
logger: Logger,
): Promise<void> {
if (logSummaryPath === undefined) {
// Failed to generate the log, so we don't expect an end summary either.
return;
}
try {
const endSummaryContent = await readFile(
this.evalLogEndSummaryPath,
"utf-8",
);
void logger.log(" --- Evaluator Log Summary --- ");
void logger.log(endSummaryContent);
} catch (e) {
void showAndLogWarningMessage(
`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`,
);
}
}
/**
* Creates the CSV file containing the results of this query. This will only be called if the query
* does not have interpreted results and the CSV file does not already exist.
@@ -675,3 +614,87 @@ export async function createInitialQueryInfo(
}),
};
}
export async function generateEvalLogSummaries(
cliServer: CodeQLCliServer,
outputDir: QueryOutputDir,
): Promise<EvaluatorLogPaths | undefined> {
const log = outputDir.evalLogPath;
if (!(await pathExists(log))) {
// No raw JSON log, so we can't generate any summaries.
return undefined;
}
let humanReadableSummary: string | undefined = undefined;
let endSummary: string | undefined = undefined;
if (await generateHumanReadableLogSummary(cliServer, outputDir)) {
humanReadableSummary = outputDir.evalLogSummaryPath;
endSummary = outputDir.evalLogEndSummaryPath;
}
let jsonSummary: string | undefined = undefined;
let summarySymbols: string | undefined = undefined;
if (isCanary()) {
// Generate JSON summary for viewer.
jsonSummary = outputDir.jsonEvalLogSummaryPath;
await cliServer.generateJsonLogSummary(log, jsonSummary);
if (humanReadableSummary !== undefined) {
summarySymbols = outputDir.evalLogSummarySymbolsPath;
await generateSummarySymbolsFile(humanReadableSummary, summarySymbols);
}
}
return {
log,
humanReadableSummary,
endSummary,
jsonSummary,
summarySymbols,
};
}
/**
* Calls the appropriate CLI command to generate a human-readable log summary.
* @param qs The query server client.
* @param outputDir The query's output directory, where all of the logs are located.
* @returns True if the summary and end summary were generated, or false if not.
*/
async function generateHumanReadableLogSummary(
cliServer: CodeQLCliServer,
outputDir: QueryOutputDir,
): Promise<boolean> {
try {
await cliServer.generateLogSummary(
outputDir.evalLogPath,
outputDir.evalLogSummaryPath,
outputDir.evalLogEndSummaryPath,
);
return true;
} catch (e) {
void showAndLogWarningMessage(
`Failed to generate human-readable structured evaluator log summary. Reason: ${getErrorMessage(
e,
)}`,
);
return false;
}
}
/**
* Logs the end summary to the Output window and log file.
* @param logSummaryPath Path to the human-readable log summary
* @param qs The query server client.
*/
export async function logEndSummary(
endSummary: string,
logger: BaseLogger,
): Promise<void> {
try {
const endSummaryContent = await readFile(endSummary, "utf-8");
void logger.log(" --- Evaluator Log Summary --- ");
void logger.log(endSummaryContent);
} catch (e) {
void showAndLogWarningMessage(
`Could not read structured evaluator log end of summary file at ${endSummary}.`,
);
}
}

9
extensions/ql-vscode/test/common/logging/output-channel-logger.test.ts
View File

@@ -1,7 +1,12 @@
import { readdirSync, readFileSync } from "fs-extra";
import { join } from "path";
import * as tmp from "tmp";
import { Logger, OutputChannelLogger, TeeLogger } from "../../../src/common";
import {
BaseLogger,
Logger,
OutputChannelLogger,
TeeLogger,
} from "../../../src/common";
jest.setTimeout(999999);
@@ -88,7 +93,7 @@ describe("OutputChannelLogger tests", function () {
function createSideLogger(
logger: Logger,
additionalLogLocation: string,
): Logger {
): BaseLogger {
return new TeeLogger(
logger,
join(tempFolders.storagePath.name, additionalLogLocation),