mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Scan logs on change in current query

Browse Source
This commit is contained in:
Dave Bartolomeo
2022-08-09 18:02:27 -04:00
parent 3c57597a19
commit 838a2b71ac
4 changed files with 166 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
extensions/ql-vscode/src/extension.ts
View File

@@ -101,6 +101,7 @@ import { RemoteQuery } from './remote-queries/remote-query';
import { EvalLogViewer } from './eval-log-viewer';
import { SummaryLanguageSupport } from './log-insights/summary-language-support';
import { JoinOrderScannerProvider } from './log-insights/join-order';
import { LogScannerService } from './log-insights/log-scanner-service';
/**
* extension.ts
@@ -485,7 +486,9 @@ async function activateWithInstalledDistribution(
ctx.subscriptions.push(qhm);
void logger.log('Initializing evaluation log scanners.');
ctx.subscriptions.push(qhm.registerLogScannerProvider(new JoinOrderScannerProvider()));
const logScannerService = new LogScannerService(qhm);
ctx.subscriptions.push(logScannerService);
ctx.subscriptions.push(logScannerService.registerLogScannerProvider(new JoinOrderScannerProvider()));
void logger.log('Reading query history');
await qhm.readQueryHistory();
@@ -520,8 +523,6 @@ async function activateWithInstalledDistribution(
forceReveal: WebviewReveal
): Promise<void> {
await intm.showResults(query, forceReveal, false);
// Always update the log warnings so they stay in sync with the results.
await qhm.scanEvalLog(query);
}
async function compileAndRunQuery(
@@ -562,7 +563,7 @@ async function activateWithInstalledDistribution(
undefined,
item,
);
item.completeThisQuery(completedQueryInfo);
qhm.completeQuery(item, completedQueryInfo);
await showResultsForCompletedQuery(item as CompletedLocalQueryInfo, WebviewReveal.NotForced);
// Note we must update the query history view after showing results as the
// display and sorting might depend on the number of results

133
extensions/ql-vscode/src/log-insights/log-scanner-service.ts Normal file
View File

@@ -0,0 +1,133 @@
import { Diagnostic, DiagnosticSeverity, Disposable, languages, Range, Uri } from 'vscode';
import { DisposableObject } from '../pure/disposable-object';
import { QueryHistoryManager } from '../query-history';
import { QueryHistoryInfo } from '../query-results';
import { EvaluationLogProblemReporter, EvaluationLogScannerProvider } from './log-scanner';
import { PipelineInfo, SummarySymbols } from './summary-parser';
import * as fs from 'fs-extra';
import { readJsonlFile } from './jsonl-reader';
/**
* Compute the key used to find a predicate in the summary symbols.
* @param name The name of the predicate.
* @param raHash The RA hash of the predicate.
* @returns The key of the predicate, consisting of `name@shortHash`, where `shortHash` is the first
* eight characters of `raHash`.
*/
function predicateSymbolKey(name: string, raHash: string): string {
return `${name}@${raHash.substring(0, 8)}`;
}
/**
* Implementation of `EvaluationLogProblemReporter` that generates `Diagnostic` objects to display
* in the VS Code "Problems" view.
*/
class ProblemReporter implements EvaluationLogProblemReporter {
public readonly diagnostics: Diagnostic[] = [];
constructor(private readonly symbols: SummarySymbols | undefined) {
}
public reportProblem(predicateName: string, raHash: string, iteration: number, message: string): void {
const nameWithHash = predicateSymbolKey(predicateName, raHash);
const predicateSymbol = this.symbols?.predicates[nameWithHash];
let predicateInfo: PipelineInfo | undefined = undefined;
if (predicateSymbol !== undefined) {
predicateInfo = predicateSymbol.iterations[iteration];
}
if (predicateInfo !== undefined) {
const range = new Range(predicateInfo.raStartLine, 0, predicateInfo.raEndLine + 1, 0);
this.diagnostics.push(new Diagnostic(range, message, DiagnosticSeverity.Error));
}
}
}
export class LogScannerService extends DisposableObject {
private readonly scannerProviders = new Map<number, EvaluationLogScannerProvider>();
private nextScannerProviderId = 0;
private readonly diagnosticCollection = this.push(languages.createDiagnosticCollection('ql-eval-log'));
private currentItem: QueryHistoryInfo | undefined = undefined;
constructor(qhm: QueryHistoryManager) {
super();
this.push(qhm.onDidChangeCurrentQueryItem(async (item) => {
if (item !== this.currentItem) {
this.currentItem = item;
await this.scanEvalLog(item);
}
}));
this.push(qhm.onDidCompleteQuery(async (item) => {
if (item === this.currentItem) {
await this.scanEvalLog(item);
}
}));
}
/**
* Scan the evaluation log for a query, and report any diagnostics.
*
* @param query The query whose log is to be scanned.
*/
public async scanEvalLog(
query: QueryHistoryInfo | undefined
): Promise<void> {
this.diagnosticCollection.clear();
if ((query === undefined) || (query.t !== 'local')) {
return;
}
if ((query !== undefined) && (query.t === 'local') && query.jsonEvalLogSummaryLocation) {
const diagnostics = await this.scanLog(query.jsonEvalLogSummaryLocation, query.evalLogSummarySymbolsLocation);
const uri = Uri.file(query.evalLogSummaryLocation!);
this.diagnosticCollection.set(uri, diagnostics);
}
}
/**
* Register a provider that can create instances of `EvaluationLogScanner` to scan evaluation logs
* for problems.
* @param provider The provider.
* @returns A `Disposable` that, when disposed, will unregister the provider.
*/
public registerLogScannerProvider(provider: EvaluationLogScannerProvider): Disposable {
const id = this.nextScannerProviderId;
this.nextScannerProviderId++;
this.scannerProviders.set(id, provider);
return {
dispose: () => {
this.scannerProviders.delete(id);
}
};
}
/**
* Scan the evaluator summary log for problems, using the scanners for all registered providers.
* @param jsonSummaryLocation The file path of the JSON summary log.
* @param symbolsLocation The file path of the symbols file for the human-readable log summary.
* @returns An array of `Diagnostic`s representing the problems found by scanners.
*/
private async scanLog(jsonSummaryLocation: string, symbolsLocation: string | undefined): Promise<Diagnostic[]> {
let symbols: SummarySymbols | undefined = undefined;
if (symbolsLocation !== undefined) {
symbols = JSON.parse(await fs.readFile(symbolsLocation, { encoding: 'utf-8' }));
}
const problemReporter = new ProblemReporter(symbols);
const scanners = [...this.scannerProviders.values()].map(p => p.createScanner(problemReporter));
await readJsonlFile(jsonSummaryLocation, async obj => {
scanners.forEach(scanner => {
scanner.onEvent(obj);
});
});
scanners.forEach(scanner => scanner.onDone());
return problemReporter.diagnostics;
}
}

137
extensions/ql-vscode/src/query-history.ts
View File

@@ -1,16 +1,15 @@
import * as path from 'path';
import {
commands,
Diagnostic,
Disposable,
env,
Event,
EventEmitter,
ExtensionContext,
languages,
ProviderResult,
Range,
ThemeIcon,
TreeDataProvider,
TreeItem,
TreeView,
Uri,
@@ -49,10 +48,7 @@ import { WebviewReveal } from './interface-utils';
import { EvalLogViewer } from './eval-log-viewer';
import EvalLogTreeBuilder from './eval-log-tree-builder';
import { EvalLogData, parseViewerData } from './pure/log-summary-parser';
import { PipelineInfo, SummarySymbols } from './log-insights/summary-parser';
import { DiagnosticSeverity } from 'vscode-languageclient';
import { EvaluationLogProblemReporter, EvaluationLogScannerProvider } from './log-insights/log-scanner';
import { readJsonlFile } from './log-insights/jsonl-reader';
import { QueryWithResults } from './run-queries';
/**
* query-history.ts
@@ -120,7 +116,7 @@ const WORKSPACE_QUERY_HISTORY_FILE = 'workspace-query-history.json';
/**
* Tree data provider for the query history view.
*/
export class HistoryTreeDataProvider extends DisposableObject {
export class HistoryTreeDataProvider extends DisposableObject implements TreeDataProvider<QueryHistoryInfo> {
private _sortOrder = SortOrder.DateAsc;
private _onDidChangeTreeData = super.push(new EventEmitter<QueryHistoryInfo | undefined>());
@@ -128,6 +124,10 @@ export class HistoryTreeDataProvider extends DisposableObject {
readonly onDidChangeTreeData: Event<QueryHistoryInfo | undefined> = this
._onDidChangeTreeData.event;
private _onDidChangeCurrentQueryItem = super.push(new EventEmitter<QueryHistoryInfo | undefined>());
public readonly onDidChangeCurrentQueryItem = this._onDidChangeCurrentQueryItem.event;
private history: QueryHistoryInfo[] = [];
private failedIconPath: string;
@@ -266,7 +266,10 @@ export class HistoryTreeDataProvider extends DisposableObject {
}
setCurrentItem(item?: QueryHistoryInfo) {
this.current = item;
if (item !== this.current) {
this.current = item;
this._onDidChangeCurrentQueryItem.fire(item);
}
}
remove(item: QueryHistoryInfo) {
@@ -292,7 +295,7 @@ export class HistoryTreeDataProvider extends DisposableObject {
set allHistory(history: QueryHistoryInfo[]) {
this.history = history;
this.current = history[0];
this.setCurrentItem(history[0]);
this.refresh();
}
@@ -310,41 +313,6 @@ export class HistoryTreeDataProvider extends DisposableObject {
}
}
/**
* Compute the key used to find a predicate in the summary symbols.
* @param name The name of the predicate.
* @param raHash The RA hash of the predicate.
* @returns The key of the predicate, consisting of `name@shortHash`, where `shortHash` is the first
* eight characters of `raHash`.
*/
function predicateSymbolKey(name: string, raHash: string): string {
return `${name}@${raHash.substring(0, 8)}`;
}
/**
* Implementation of `EvaluationLogProblemReporter` that generates `Diagnostic` objects to display
* in the VS Code "Problems" view.
*/
class ProblemReporter implements EvaluationLogProblemReporter {
public readonly diagnostics: Diagnostic[] = [];
constructor(private readonly symbols: SummarySymbols | undefined) {
}
public reportProblem(predicateName: string, raHash: string, iteration: number, message: string): void {
const nameWithHash = predicateSymbolKey(predicateName, raHash);
const predicateSymbol = this.symbols?.predicates[nameWithHash];
let predicateInfo: PipelineInfo | undefined = undefined;
if (predicateSymbol !== undefined) {
predicateInfo = predicateSymbol.iterations[iteration];
}
if (predicateInfo !== undefined) {
const range = new Range(predicateInfo.raStartLine, 0, predicateInfo.raEndLine + 1, 0);
this.diagnostics.push(new Diagnostic(range, message, DiagnosticSeverity.Error));
}
}
}
export class QueryHistoryManager extends DisposableObject {
treeDataProvider: HistoryTreeDataProvider;
@@ -352,7 +320,6 @@ export class QueryHistoryManager extends DisposableObject {
lastItemClick: { time: Date; item: QueryHistoryInfo } | undefined;
compareWithItem: LocalQueryInfo | undefined;
queryHistoryScrubber: Disposable | undefined;
private readonly diagnosticCollection = this.push(languages.createDiagnosticCollection('ql-eval-log'));
private queryMetadataStorageLocation;
private readonly _onDidAddQueryItem = super.push(new EventEmitter<QueryHistoryInfo>());
@@ -367,8 +334,11 @@ export class QueryHistoryManager extends DisposableObject {
readonly onWillOpenQueryItem: Event<QueryHistoryInfo> = this
._onWillOpenQueryItem.event;
private readonly scannerProviders = new Map<number, EvaluationLogScannerProvider>();
private nextScannerProviderId = 0;
private readonly _onDidChangeCurrentQueryItem = super.push(new EventEmitter<QueryHistoryInfo | undefined>());
readonly onDidChangeCurrentQueryItem = this._onDidChangeCurrentQueryItem.event;
private readonly _onDidCompleteQuery = super.push(new EventEmitter<LocalQueryInfo>());
readonly onDidCompleteQuery = this._onDidCompleteQuery.event;
constructor(
private readonly qs: QueryServerClient,
@@ -402,6 +372,11 @@ export class QueryHistoryManager extends DisposableObject {
canSelectMany: true,
}));
// Forward any change of current history item from the tree data.
this.push(this.treeDataProvider.onDidChangeCurrentQueryItem((item) => {
this._onDidChangeCurrentQueryItem.fire(item);
}));
// Lazily update the tree view selection due to limitations of TreeView API (see
// `updateTreeViewSelectionIfVisible` doc for details)
this.push(
@@ -594,6 +569,11 @@ export class QueryHistoryManager extends DisposableObject {
this.registerToRemoteQueriesEvents();
}
public completeQuery(info: LocalQueryInfo, results: QueryWithResults): void {
info.completeThisQuery(results);
this._onDidCompleteQuery.fire(info);
}
private getCredentials() {
return Credentials.initialize(this.ctx);
}
@@ -1017,24 +997,6 @@ export class QueryHistoryManager extends DisposableObject {
}
}
/**
* Scan the evaluation log for a query, and report any diagnostics.
*
* @param query The query whose log is to be scanned.
*/
public async scanEvalLog(
query: LocalQueryInfo
): Promise<void> {
this.diagnosticCollection.clear();
if (query.jsonEvalLogSummaryLocation) {
const diagnostics = await this.scanLog(query.jsonEvalLogSummaryLocation, query.evalLogSummarySymbolsLocation);
const uri = Uri.file(query.evalLogSummaryLocation!);
this.diagnosticCollection.set(uri, diagnostics);
} else {
this.warnNoEvalLogs();
}
}
async handleCancel(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[]
@@ -1209,51 +1171,6 @@ export class QueryHistoryManager extends DisposableObject {
this.updateTreeViewSelectionIfVisible();
}
/**
* Register a provider that can create instances of `EvaluationLogScanner` to scan evaluation logs
* for problems.
* @param provider The provider.
* @returns A `Disposable` that, when disposed, will unregister the provider.
*/
registerLogScannerProvider(provider: EvaluationLogScannerProvider): Disposable {
const id = this.nextScannerProviderId;
this.nextScannerProviderId++;
this.scannerProviders.set(id, provider);
return {
dispose: () => {
this.scannerProviders.delete(id);
}
};
}
/**
* Scan the evaluator summary log for problems, using the scanners for all registered providers.
* @param jsonSummaryLocation The file path of the JSON summary log.
* @param symbolsLocation The file path of the symbols file for the human-readable log summary.
* @returns An array of `Diagnostic`s representing the problems found by scanners.
*/
private async scanLog(jsonSummaryLocation: string, symbolsLocation: string | undefined): Promise<Diagnostic[]> {
let symbols: SummarySymbols | undefined = undefined;
if (symbolsLocation !== undefined) {
symbols = JSON.parse(await fs.readFile(symbolsLocation, { encoding: 'utf-8' }));
}
const problemReporter = new ProblemReporter(symbols);
const scanners = [...this.scannerProviders.values()].map(p => p.createScanner(problemReporter));
await readJsonlFile(jsonSummaryLocation, async obj => {
scanners.forEach(scanner => {
scanner.onEvent(obj);
});
});
scanners.forEach(scanner => scanner.onDone());
return problemReporter.diagnostics;
}
/**
* Update the tree view selection if the tree view is visible.
*

2
extensions/ql-vscode/src/query-results.ts
View File

@@ -283,7 +283,7 @@ export class LocalQueryInfo {
return !!this.completedQuery;
}
completeThisQuery(info: QueryWithResults) {
completeThisQuery(info: QueryWithResults): void {
this.completedQuery = new CompletedQueryInfo(info);
// dispose of the cancellation token source and also ensure the source is not serialized as JSON