mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #2169 from github/dbartol/tee-logger

Browse Source 
Use `TeeLogger` instead of `additionalLogLocation:` option
This commit is contained in:
Dave Bartolomeo
2023-03-13 17:49:13 -04:00
committed by GitHub
parent 205327d8aa a1d8aac391
commit 6c3698bfb3
19 changed files with 123 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
extensions/ql-vscode/src/common/logging/index.ts
View File

@@ -1,3 +1,4 @@
export * from "./logger";
export * from "./tee-logger";
export * from "./vscode/loggers";
export * from "./vscode/output-channel-logger";

10
extensions/ql-vscode/src/common/logging/logger.ts
View File

@@ -1,9 +1,6 @@
export interface LogOptions {
// If false, don't output a trailing newline for the log entry. Default true.
trailingNewline?: boolean;
// If specified, add this log entry to the log file at the specified location.
additionalLogLocation?: string;
}
export interface Logger {
@@ -25,11 +22,4 @@ export interface Logger {
* @param preserveFocus When `true` the channel will not take focus.
*/
show(preserveFocus?: boolean): void;
/**
* Remove the log at the specified location.
*
* @param location log to remove
*/
removeAdditionalLogLocation(location: string | undefined): void;
}

68
extensions/ql-vscode/src/common/logging/tee-logger.ts Normal file
View File

@@ -0,0 +1,68 @@
import { appendFile, ensureFile } from "fs-extra";
import { isAbsolute } from "path";
import { getErrorMessage } from "../../pure/helpers-pure";
import { Logger, LogOptions } from "./logger";
/**
* An implementation of {@link Logger} that sends the output both to another {@link Logger}
* and to a file.
*
* The first time a message is written, an additional banner is written to the underlying logger
* pointing the user to the "side log" file.
*/
export class TeeLogger implements Logger {
private emittedRedirectMessage = false;
private error = false;
public constructor(
private readonly logger: Logger,
private readonly location: string,
) {
if (!isAbsolute(location)) {
throw new Error(
`Additional Log Location must be an absolute path: ${location}`,
);
}
}
async log(message: string, options = {} as LogOptions): Promise<void> {
if (!this.emittedRedirectMessage) {
this.emittedRedirectMessage = true;
const msg = `| Log being saved to ${this.location} |`;
const separator = new Array(msg.length).fill("-").join("");
await this.logger.log(separator);
await this.logger.log(msg);
await this.logger.log(separator);
}
if (!this.error) {
try {
const trailingNewline = options.trailingNewline ?? true;
await ensureFile(this.location);
await appendFile(
this.location,
message + (trailingNewline ? "\n" : ""),
{
encoding: "utf8",
},
);
} catch (e) {
// Write an error message to the primary log, and stop trying to write to the side log.
this.error = true;
const errorMessage = getErrorMessage(e);
await this.logger.log(
`Error writing to additional log file: ${errorMessage}`,
);
}
}
if (!this.error) {
await this.logger.log(message, options);
}
}
show(preserveFocus?: boolean): void {
this.logger.show(preserveFocus);
}
}

52
extensions/ql-vscode/src/common/logging/vscode/output-channel-logger.ts
View File

@@ -1,6 +1,4 @@
import { window as Window, OutputChannel, Progress } from "vscode";
import { ensureFile, appendFile } from "fs-extra";
import { isAbsolute } from "path";
import { Logger, LogOptions } from "../logger";
import { DisposableObject } from "../../../pure/disposable-object";
@@ -9,10 +7,6 @@ import { DisposableObject } from "../../../pure/disposable-object";
*/
export class OutputChannelLogger extends DisposableObject implements Logger {
public readonly outputChannel: OutputChannel;
private readonly additionalLocations = new Map<
string,
AdditionalLogLocation
>();
isCustomLogDirectory: boolean;
constructor(title: string) {
@@ -32,27 +26,6 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
} else {
this.outputChannel.append(message);
}
if (options.additionalLogLocation) {
if (!isAbsolute(options.additionalLogLocation)) {
throw new Error(
`Additional Log Location must be an absolute path: ${options.additionalLogLocation}`,
);
}
const logPath = options.additionalLogLocation;
let additional = this.additionalLocations.get(logPath);
if (!additional) {
const msg = `| Log being saved to ${logPath} |`;
const separator = new Array(msg.length).fill("-").join("");
this.outputChannel.appendLine(separator);
this.outputChannel.appendLine(msg);
this.outputChannel.appendLine(separator);
additional = new AdditionalLogLocation(logPath);
this.additionalLocations.set(logPath, additional);
}
await additional.log(message, options);
}
} catch (e) {
if (e instanceof Error && e.message === "Channel has been closed") {
// Output channel is closed logging to console instead
@@ -69,31 +42,6 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
show(preserveFocus?: boolean): void {
this.outputChannel.show(preserveFocus);
}
removeAdditionalLogLocation(location: string | undefined): void {
if (location) {
this.additionalLocations.delete(location);
}
}
}
class AdditionalLogLocation {
constructor(private location: string) {}
async log(message: string, options = {} as LogOptions): Promise<void> {
if (options.trailingNewline === undefined) {
options.trailingNewline = true;
}
await ensureFile(this.location);
await appendFile(
this.location,
message + (options.trailingNewline ? "\n" : ""),
{
encoding: "utf8",
},
);
}
}
export type ProgressReporter = Progress<{ message: string }>;

27
extensions/ql-vscode/src/legacy-query-server/queryserver-client.ts
View File

@@ -1,4 +1,3 @@
import { dirname } from "path";
import { ensureFile } from "fs-extra";
import { DisposableObject } from "../pure/disposable-object";
@@ -13,10 +12,8 @@ import {
progress,
ProgressMessage,
WithProgressId,
compileQuery,
} from "../pure/legacy-messages";
import { ProgressCallback, ProgressTask } from "../commandRunner";
import { findQueryLogFile } from "../run-queries-shared";
import { ServerProcess } from "../json-rpc-server";
type WithProgressReporting = (
@@ -56,7 +53,7 @@ export class QueryServerClient extends DisposableObject {
this.queryServerStartListeners.push(e);
};
public activeQueryLogFile: string | undefined;
public activeQueryLogger: Logger;
constructor(
readonly config: QueryServerConfig,
@@ -65,6 +62,9 @@ export class QueryServerClient extends DisposableObject {
withProgressReporting: WithProgressReporting,
) {
super();
// Since no query is active when we initialize, just point the "active query logger" to the
// default logger.
this.activeQueryLogger = this.logger;
// When the query server configuration changes, restart the query server.
if (config.onDidChangeConfiguration !== undefined) {
this.push(
@@ -177,9 +177,8 @@ export class QueryServerClient extends DisposableObject {
args,
this.logger,
(data) =>
this.logger.log(data.toString(), {
this.activeQueryLogger.log(data.toString(), {
trailingNewline: false,
additionalLogLocation: this.activeQueryLogFile,
}),
undefined, // no listener for stdout
progressReporter,
@@ -240,8 +239,6 @@ export class QueryServerClient extends DisposableObject {
): Promise<R> {
const id = this.nextProgress++;
this.progressCallbacks[id] = progress;
this.updateActiveQuery(type.method, parameter);
try {
if (this.serverProcess === undefined) {
throw new Error("No query server process found.");
@@ -255,18 +252,4 @@ export class QueryServerClient extends DisposableObject {
delete this.progressCallbacks[id];
}
}
/**
* Updates the active query every time there is a new request to compile.
* The active query is used to specify the side log.
*
* This isn't ideal because in situations where there are queries running
* in parallel, each query's log messages are interleaved. Fixing this
* properly will require a change in the query server.
*/
private updateActiveQuery(method: string, parameter: any): void {
if (method === compileQuery.method) {
this.activeQueryLogFile = findQueryLogFile(dirname(parameter.resultPath));
}
}
}

38
extensions/ql-vscode/src/legacy-query-server/run-queries.ts
View File

@@ -15,7 +15,7 @@ import {
} from "../helpers";
import { ProgressCallback } from "../commandRunner";
import { QueryMetadata } from "../pure/interface-types";
import { extLogger } from "../common";
import { extLogger, Logger, TeeLogger } from "../common";
import * as messages from "../pure/legacy-messages";
import { InitialQueryInfo, LocalQueryInfo } from "../query-results";
import * as qsClient from "./queryserver-client";
@@ -66,7 +66,8 @@ export class QueryInProgress {
dbItem: DatabaseItem,
progress: ProgressCallback,
token: CancellationToken,
queryInfo?: LocalQueryInfo,
logger: Logger,
queryInfo: LocalQueryInfo | undefined,
): Promise<messages.EvaluationResult> {
if (!dbItem.contents || dbItem.error) {
throw new Error("Can't run query on invalid database.");
@@ -137,7 +138,7 @@ export class QueryInProgress {
await this.queryEvalInfo.addQueryLogs(
queryInfo,
qs.cliServer,
qs.logger,
logger,
);
} else {
void showAndLogWarningMessage(
@@ -162,6 +163,7 @@ export class QueryInProgress {
program: messages.QlProgram,
progress: ProgressCallback,
token: CancellationToken,
logger: Logger,
): Promise<messages.CompilationMessage[]> {
let compiled: messages.CheckQueryResult | undefined;
try {
@@ -190,6 +192,11 @@ export class QueryInProgress {
target,
};
// Update the active query logger every time there is a new request to compile.
// This isn't ideal because in situations where there are queries running
// in parallel, each query's log messages are interleaved. Fixing this
// properly will require a change in the query server.
qs.activeQueryLogger = logger;
compiled = await qs.sendRequest(
messages.compileQuery,
params,
@@ -197,9 +204,7 @@ export class QueryInProgress {
progress,
);
} finally {
void qs.logger.log(" - - - COMPILATION DONE - - - ", {
additionalLogLocation: this.queryEvalInfo.logPath,
});
void logger.log(" - - - COMPILATION DONE - - - ");
}
return (compiled?.messages || []).filter(
(msg) => msg.severity === messages.Severity.ERROR,
@@ -386,6 +391,8 @@ export async function compileAndRunQueryAgainstDatabase(
metadata,
templates,
);
const logger = new TeeLogger(qs.logger, query.queryEvalInfo.logPath);
await query.queryEvalInfo.createTimestampFile();
let upgradeDir: tmp.DirectoryResult | undefined;
@@ -402,7 +409,7 @@ export async function compileAndRunQueryAgainstDatabase(
);
let errors;
try {
errors = await query.compile(qs, qlProgram, progress, token);
errors = await query.compile(qs, qlProgram, progress, token, logger);
} catch (e) {
if (
e instanceof ResponseError &&
@@ -422,6 +429,7 @@ export async function compileAndRunQueryAgainstDatabase(
dbItem,
progress,
token,
logger,
queryInfo,
);
if (result.resultType !== messages.QueryResultType.SUCCESS) {
@@ -439,18 +447,14 @@ export async function compileAndRunQueryAgainstDatabase(
result,
successful: result.resultType === messages.QueryResultType.SUCCESS,
logFileLocation: result.logFileLocation,
dispose: () => {
qs.logger.removeAdditionalLogLocation(result.logFileLocation);
},
};
} else {
// Error dialogs are limited in size and scrollability,
// so we include a general description of the problem,
// and direct the user to the output window for the detailed compilation messages.
// However we don't show quick eval errors there so we need to display them anyway.
void qs.logger.log(
void logger.log(
`Failed to compile query ${initialInfo.queryPath} against database scheme ${qlProgram.dbschemePath}:`,
{ additionalLogLocation: query.queryEvalInfo.logPath },
);
const formattedMessages: string[] = [];
@@ -459,9 +463,7 @@ export async function compileAndRunQueryAgainstDatabase(
const message = error.message || "[no error message available]";
const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
formattedMessages.push(formatted);
void qs.logger.log(formatted, {
additionalLogLocation: query.queryEvalInfo.logPath,
});
void logger.log(formatted);
}
if (initialInfo.isQuickEval && formattedMessages.length <= 2) {
// If there are more than 2 error messages, they will not be displayed well in a popup
@@ -484,9 +486,8 @@ export async function compileAndRunQueryAgainstDatabase(
try {
await upgradeDir?.cleanup();
} catch (e) {
void qs.logger.log(
void logger.log(
`Could not clean up the upgrades dir. Reason: ${getErrorMessage(e)}`,
{ additionalLogLocation: query.queryEvalInfo.logPath },
);
}
}
@@ -535,9 +536,6 @@ function createSyntheticResult(
runId: 0,
},
successful: false,
dispose: () => {
/**/
},
};
}

2
extensions/ql-vscode/src/query-history/query-history-manager.ts
View File

@@ -556,7 +556,6 @@ export class QueryHistoryManager extends DisposableObject {
!(await pathExists(item.completedQuery?.query.querySaveDir))
) {
this.treeDataProvider.remove(item);
item.completedQuery?.dispose();
}
}),
);
@@ -577,7 +576,6 @@ export class QueryHistoryManager extends DisposableObject {
// Removing in progress local queries is not supported. They must be cancelled first.
if (item.status !== QueryStatus.InProgress) {
this.treeDataProvider.remove(item);
item.completedQuery?.dispose();
// User has explicitly asked for this query to be removed.
// We need to delete it from disk as well.

9
extensions/ql-vscode/src/query-results.ts
View File

@@ -55,11 +55,6 @@ export class CompletedQueryInfo implements QueryWithResults {
readonly logFileLocation?: string;
resultCount: number;
/**
* This dispose method is called when the query is removed from the history view.
*/
dispose: () => void;
/**
* Map from result set name to SortedResultSetInfo.
*/
@@ -85,10 +80,6 @@ export class CompletedQueryInfo implements QueryWithResults {
this.message = evaluation.message;
this.successful = evaluation.successful;
// Use the dispose method from the evaluation.
// The dispose will clean up any additional log locations that this
// query may have created.
this.dispose = evaluation.dispose;
this.sortedResultsInfo = {};
this.resultCount = 0;

4
extensions/ql-vscode/src/query-serialization.ts
View File

@@ -55,10 +55,6 @@ export async function deserializeQueryHistory(
q.completedQuery.query,
QueryEvaluationInfo.prototype,
);
// deserialized queries do not need to be disposed
q.completedQuery.dispose = () => {
/**/
};
// Previously, there was a typo in the completedQuery type. There was a field
// `sucessful` and it was renamed to `successful`. We need to handle this case.

28
extensions/ql-vscode/src/query-server/queryserver-client.ts
View File

@@ -1,4 +1,3 @@
import { dirname } from "path";
import { ensureFile } from "fs-extra";
import { DisposableObject } from "../pure/disposable-object";
@@ -12,9 +11,7 @@ import {
ProgressMessage,
WithProgressId,
} from "../pure/new-messages";
import * as messages from "../pure/new-messages";
import { ProgressCallback, ProgressTask } from "../commandRunner";
import { findQueryLogFile } from "../run-queries-shared";
import { ServerProcess } from "../json-rpc-server";
type ServerOpts = {
@@ -53,7 +50,7 @@ export class QueryServerClient extends DisposableObject {
this.queryServerStartListeners.push(e);
};
public activeQueryLogFile: string | undefined;
public activeQueryLogger: Logger;
constructor(
readonly config: QueryServerConfig,
@@ -62,6 +59,9 @@ export class QueryServerClient extends DisposableObject {
withProgressReporting: WithProgressReporting,
) {
super();
// Since no query is active when we initialize, just point the "active query logger" to the
// default logger.
this.activeQueryLogger = this.logger;
// When the query server configuration changes, restart the query server.
if (config.onDidChangeConfiguration !== undefined) {
this.push(
@@ -167,9 +167,8 @@ export class QueryServerClient extends DisposableObject {
args,
this.logger,
(data) =>
this.logger.log(data.toString(), {
this.activeQueryLogger.log(data.toString(), {
trailingNewline: false,
additionalLogLocation: this.activeQueryLogFile,
}),
undefined, // no listener for stdout
progressReporter,
@@ -210,7 +209,6 @@ export class QueryServerClient extends DisposableObject {
const id = this.nextProgress++;
this.progressCallbacks[id] = progress;
this.updateActiveQuery(type.method, parameter);
try {
if (this.serverProcess === undefined) {
throw new Error("No query server process found.");
@@ -224,20 +222,4 @@ export class QueryServerClient extends DisposableObject {
delete this.progressCallbacks[id];
}
}
/**
* Updates the active query every time there is a new request to compile.
* The active query is used to specify the side log.
*
* This isn't ideal because in situations where there are queries running
* in parallel, each query's log messages are interleaved. Fixing this
* properly will require a change in the query server.
*/
private updateActiveQuery(method: string, parameter: any): void {
if (method === messages.runQuery.method) {
this.activeQueryLogFile = findQueryLogFile(
dirname(dirname((parameter as messages.RunQueryParams).outputPath)),
);
}
}
}

13
extensions/ql-vscode/src/query-server/run-queries.ts
View File

@@ -9,7 +9,7 @@ import {
showAndLogWarningMessage,
tryGetQueryMetadata,
} from "../helpers";
import { extLogger } from "../common";
import { extLogger, TeeLogger } from "../common";
import * as messages from "../pure/new-messages";
import { QueryResultType } from "../pure/legacy-messages";
import { InitialQueryInfo, LocalQueryInfo } from "../query-results";
@@ -88,9 +88,15 @@ export async function compileAndRunQueryAgainstDatabase(
target,
extensionPacks,
};
const logger = new TeeLogger(qs.logger, query.logPath);
await query.createTimestampFile();
let result: messages.RunQueryResult | undefined;
try {
// Update the active query logger every time there is a new request to compile.
// This isn't ideal because in situations where there are queries running
// in parallel, each query's log messages are interleaved. Fixing this
// properly will require a change in the query server.
qs.activeQueryLogger = logger;
result = await qs.sendRequest(
messages.runQuery,
queryToRun,
@@ -105,7 +111,7 @@ export async function compileAndRunQueryAgainstDatabase(
} finally {
if (queryInfo) {
if (await query.hasEvalLog()) {
await query.addQueryLogs(queryInfo, qs.cliServer, qs.logger);
await query.addQueryLogs(queryInfo, qs.cliServer, logger);
} else {
void showAndLogWarningMessage(
`Failed to write structured evaluator log to ${query.evalLogPath}.`,
@@ -160,8 +166,5 @@ export async function compileAndRunQueryAgainstDatabase(
},
message,
successful,
dispose: () => {
qs.logger.removeAdditionalLogLocation(undefined);
},
};
}

9
extensions/ql-vscode/src/run-queries-shared.ts
View File

@@ -298,12 +298,8 @@ export class QueryEvaluationInfo {
this.evalLogEndSummaryPath,
"utf-8",
);
void logger.log(" --- Evaluator Log Summary --- ", {
additionalLogLocation: this.logPath,
});
void logger.log(endSummaryContent, {
additionalLogLocation: this.logPath,
});
void logger.log(" --- Evaluator Log Summary --- ");
void logger.log(endSummaryContent);
} catch (e) {
void showAndLogWarningMessage(
`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`,
@@ -436,7 +432,6 @@ export class QueryEvaluationInfo {
export interface QueryWithResults {
readonly query: QueryEvaluationInfo;
readonly logFileLocation?: string;
readonly dispose: () => void;
readonly successful?: boolean;
readonly message?: string;
readonly result: legacyMessages.EvaluationResult;

1
extensions/ql-vscode/test/__mocks__/loggerMock.ts
View File

@@ -4,6 +4,5 @@ export function createMockLogger(): Logger {
return {
log: jest.fn(() => Promise.resolve()),
show: jest.fn(),
removeAdditionalLogLocation: jest.fn(),
};
}

30
extensions/ql-vscode/test/common/logging/output-channel-logger.test.ts
View File

@@ -1,7 +1,7 @@
import { readdirSync, readFileSync } from "fs-extra";
import { join } from "path";
import * as tmp from "tmp";
import { OutputChannelLogger } from "../../../src/common";
import { Logger, OutputChannelLogger, TeeLogger } from "../../../src/common";
jest.setTimeout(999999);
@@ -58,16 +58,19 @@ describe("OutputChannelLogger tests", function () {
expect(mockOutputChannel.appendLine).not.toBeCalledWith("yyy");
expect(mockOutputChannel.append).toBeCalledWith("yyy");
await logger.log("zzz", createLogOptions("hucairz"));
const hucairz = createSideLogger(logger, "hucairz");
await hucairz.log("zzz");
// should have created 1 side log
expect(readdirSync(tempFolders.storagePath.name)).toEqual(["hucairz"]);
});
it("should create a side log", async () => {
await logger.log("xxx", createLogOptions("first"));
await logger.log("yyy", createLogOptions("second"));
await logger.log("zzz", createLogOptions("first", false));
const first = createSideLogger(logger, "first");
await first.log("xxx");
const second = createSideLogger(logger, "second");
await second.log("yyy");
await first.log("zzz", { trailingNewline: false });
await logger.log("aaa");
// expect 2 side logs
@@ -82,16 +85,13 @@ describe("OutputChannelLogger tests", function () {
).toBe("yyy\n");
});
function createLogOptions(
function createSideLogger(
logger: Logger,
additionalLogLocation: string,
trailingNewline?: boolean,
) {
return {
additionalLogLocation: join(
tempFolders.storagePath.name,
additionalLogLocation,
),
trailingNewline,
};
): Logger {
return new TeeLogger(
logger,
join(tempFolders.storagePath.name, additionalLogLocation),
);
}
});

2
extensions/ql-vscode/test/factories/query-history/local-query-history-item.ts
View File

@@ -74,7 +74,6 @@ export function createMockQueryWithResults({
hasInterpretedResults?: boolean;
hasMetadata?: boolean;
}): QueryWithResults {
const dispose = jest.fn();
const deleteQuery = jest.fn();
const metadata = hasMetadata
? ({ name: "query-name" } as QueryMetadata)
@@ -87,7 +86,6 @@ export function createMockQueryWithResults({
metadata,
} as unknown as QueryEvaluationInfo,
successful: didRunSuccessfully,
dispose,
result: {
evaluationTime: 1,
queryId: 0,

2
extensions/ql-vscode/test/vscode-tests/no-workspace/query-history/query-history-manager.test.ts
View File

@@ -344,7 +344,6 @@ describe("QueryHistoryManager", () => {
});
it("should remove the item", () => {
expect(toDelete.completedQuery!.dispose).toBeCalledTimes(1);
expect(queryHistoryManager.treeDataProvider.allHistory).toEqual(
expect.not.arrayContaining([toDelete]),
);
@@ -387,7 +386,6 @@ describe("QueryHistoryManager", () => {
});
it("should remove the item", () => {
expect(toDelete.completedQuery!.dispose).toBeCalledTimes(1);
expect(queryHistoryManager.treeDataProvider.allHistory).toEqual(
expect.not.arrayContaining([toDelete]),
);

1
extensions/ql-vscode/test/vscode-tests/no-workspace/query-results.test.ts
View File

@@ -470,7 +470,6 @@ describe("query-results", () => {
query: query.queryEvalInfo,
successful: didRunSuccessfully,
message: "foo",
dispose: jest.fn(),
result: {
evaluationTime: 1,
queryId: 0,

1
extensions/ql-vscode/test/vscode-tests/no-workspace/query-serialization.test.ts
View File

@@ -259,7 +259,6 @@ describe("serialize and deserialize", () => {
query: query.queryEvalInfo,
successful: didRunSuccessfully,
message: "foo",
dispose: jest.fn(),
result: {
evaluationTime: 1,
queryId: 0,

1
extensions/ql-vscode/test/vscode-tests/no-workspace/run-queries.test.ts
View File

@@ -190,6 +190,7 @@ describe("run-queries", () => {
mockQlProgram,
mockProgress as any,
mockCancel as any,
qs.logger,
);
expect(results).toEqual([{ message: "err", severity: Severity.ERROR }]);