From 7f65122adb82fd078d3bce3e280ede682f39988d Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Tue, 4 Apr 2023 13:54:35 +0200 Subject: [PATCH 01/25] Add generating of flow model to data extension editor This adds the automatic generation of sources/sinks/summary flows to the data extension editor using the flow model queries. This is based on the Python script available in the CodeQL repo. See: https://github.com/github/codeql/blob/main/java/ql/src/utils/modelgenerator/GenerateFlowModel.py --- .../data-extensions-editor-module.ts | 4 + .../data-extensions-editor-view.ts | 92 ++++++++++- .../generate-flow-model.ts | 146 ++++++++++++++++++ .../src/data-extensions-editor/yaml.ts | 2 +- extensions/ql-vscode/src/extension.ts | 1 + .../ql-vscode/src/pure/interface-types.ts | 16 +- .../DataExtensionsEditor.tsx | 26 ++++ 7 files changed, 282 insertions(+), 5 deletions(-) create mode 100644 extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts index 2cbf6a53d..94a1620e8 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts @@ -5,10 +5,12 @@ import { CodeQLCliServer } from "../cli"; import { QueryRunner } from "../queryRunner"; import { DatabaseManager } from "../local-databases"; import { extLogger } from "../common"; +import { App } from "../common/app"; export class DataExtensionsEditorModule { public constructor( private readonly ctx: ExtensionContext, + private readonly app: App, private readonly databaseManager: DatabaseManager, private readonly cliServer: CodeQLCliServer, private readonly queryRunner: QueryRunner, @@ -26,6 +28,8 @@ export class DataExtensionsEditorModule { const view = new DataExtensionsEditorView( this.ctx, + this.app, + this.databaseManager, this.cliServer, this.queryRunner, this.queryStorageDir, diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 066b7aaec..2458bcb47 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -18,9 +18,13 @@ import { file } from "tmp-promise"; import { readFile, writeFile } from "fs-extra"; import { dump, load } from "js-yaml"; import { getOnDiskWorkspaceFolders } from "../helpers"; -import { DatabaseItem } from "../local-databases"; +import { DatabaseItem, DatabaseManager } from "../local-databases"; import { CodeQLCliServer } from "../cli"; -import { assertNever } from "../pure/helpers-pure"; +import { assertNever, getErrorMessage } from "../pure/helpers-pure"; +import { generateFlowModel } from "./generate-flow-model"; +import { ModeledMethod } from "./interface"; +import { promptImportGithubDatabase } from "../databaseFetcher"; +import { App } from "../common/app"; export class DataExtensionsEditorView extends AbstractWebview< ToDataExtensionsEditorMessage, @@ -28,6 +32,8 @@ export class DataExtensionsEditorView extends AbstractWebview< > { public constructor( ctx: ExtensionContext, + private readonly app: App, + private readonly databaseManager: DatabaseManager, private readonly cliServer: CodeQLCliServer, private readonly queryRunner: QueryRunner, private readonly queryStorageDir: string, @@ -69,6 +75,10 @@ export class DataExtensionsEditorView extends AbstractWebview< await this.saveYaml(msg.yaml); await this.loadExternalApiUsages(); + break; + case "generateExternalApi": + await this.generateExternalApi(); + break; default: assertNever(msg); @@ -149,6 +159,84 @@ export class DataExtensionsEditorView extends AbstractWebview< await this.clearProgress(); } + protected async generateExternalApi(): Promise { + const tokenSource = new CancellationTokenSource(); + + const selectedDatabase = this.databaseManager.currentDatabaseItem; + + const database = await promptImportGithubDatabase( + this.app.commands, + this.databaseManager, + this.app.workspaceStoragePath ?? this.app.globalStoragePath, + this.app.credentials, + (update) => this.showProgress(update), + tokenSource.token, + this.cliServer, + ); + if (!database) { + await this.clearProgress(); + void extLogger.log("No database chosen"); + + return; + } + + await this.databaseManager.setCurrentDatabaseItem(selectedDatabase); + + const workspaceFolder = workspace.workspaceFolders?.find( + (folder) => folder.name === "ql", + ); + if (!workspaceFolder) { + void extLogger.log("No workspace folder 'ql' found"); + + return; + } + + await this.showProgress({ + step: 0, + maxStep: 4000, + message: "Generating external API", + }); + + try { + await generateFlowModel( + this.cliServer, + this.queryRunner, + this.queryStorageDir, + workspaceFolder.uri.fsPath, + database, + async (results) => { + const modeledMethodsByName: Record = {}; + + for (const result of results) { + modeledMethodsByName[result[0]] = result[1]; + } + + await this.postMessage({ + t: "addModeledMethods", + modeledMethods: modeledMethodsByName, + }); + }, + (update) => this.showProgress(update), + tokenSource.token, + ); + } catch (e: unknown) { + void extLogger.log(`Error: ${getErrorMessage(e)}`); + } + + await this.databaseManager.removeDatabaseItem( + () => + this.showProgress({ + step: 3900, + maxStep: 4000, + message: "Removing temporary database", + }), + tokenSource.token, + database, + ); + + await this.clearProgress(); + } + private async runQuery(): Promise { const qlpacks = await qlpackOfDatabase(this.cliServer, this.databaseItem); diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts new file mode 100644 index 000000000..d60951817 --- /dev/null +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -0,0 +1,146 @@ +import { CancellationToken } from "vscode"; +import { DatabaseItem } from "../local-databases"; +import { ModeledMethod, ModeledMethodType } from "./interface"; +import { join } from "path"; +import { QueryRunner } from "../queryRunner"; +import { CodeQLCliServer } from "../cli"; +import { extLogger, TeeLogger } from "../common"; +import { definitions } from "./yaml"; +import { ProgressCallback } from "../progress"; +import { getOnDiskWorkspaceFolders } from "../helpers"; + +class FlowModelGenerator { + constructor( + private readonly cli: CodeQLCliServer, + private readonly queryRunner: QueryRunner, + private readonly queryStorageDir: string, + private readonly qlDir: string, + private readonly databaseItem: DatabaseItem, + private readonly progress: ProgressCallback, + private readonly token: CancellationToken, + ) {} + + async getAddsTo( + type: Exclude, + queryName: string, + queryStep: number, + ): Promise | undefined> { + const definition = definitions[type]; + + const query = join( + this.qlDir, + this.databaseItem.language, + "ql/src/utils/modelgenerator", + queryName, + ); + + const queryRun = this.queryRunner.createQueryRun( + this.databaseItem.databaseUri.fsPath, + { queryPath: query, quickEvalPosition: undefined }, + false, + getOnDiskWorkspaceFolders(), + undefined, + this.queryStorageDir, + undefined, + undefined, + ); + + const queryResult = await queryRun.evaluate( + ({ step, message }) => + this.progress({ + message: `Generating ${type} model: ${message}`, + step: queryStep * 1000 + step, + maxStep: 4000, + }), + this.token, + new TeeLogger(this.queryRunner.logger, queryRun.outputDir.logPath), + ); + + const bqrsPath = queryResult.outputDir.bqrsPath; + + const bqrsInfo = await this.cli.bqrsInfo(bqrsPath); + if (bqrsInfo["result-sets"].length !== 1) { + void extLogger.log( + `Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`, + ); + return undefined; + } + + const resultSet = bqrsInfo["result-sets"][0]; + + const decodedResults = await this.cli.bqrsDecode(bqrsPath, resultSet.name); + + const results = decodedResults.tuples; + + return results + .map((result): [string, ModeledMethod] | undefined => { + const row = result[0] as string; + + return definition.readModeledMethod(row.split(";")); + }) + .filter( + (result): result is [string, ModeledMethod] => result !== undefined, + ); + } + + async run( + onResults: ( + results: Array<[string, ModeledMethod]>, + ) => void | Promise, + ) { + const summaryResults = await this.getAddsTo( + "summary", + "CaptureSummaryModels.ql", + 0, + ); + if (summaryResults) { + await onResults(summaryResults); + } + + const sinkResults = await this.getAddsTo("sink", "CaptureSinkModels.ql", 1); + if (sinkResults) { + await onResults(sinkResults); + } + + const sourceResults = await this.getAddsTo( + "source", + "CaptureSourceModels.ql", + 2, + ); + if (sourceResults) { + await onResults(sourceResults); + } + + const neutralResults = await this.getAddsTo( + "neutral", + "CaptureNeutralModels.ql", + 3, + ); + if (neutralResults) { + await onResults(neutralResults); + } + } +} + +export async function generateFlowModel( + cli: CodeQLCliServer, + queryRunner: QueryRunner, + queryStorageDir: string, + qlDir: string, + databaseItem: DatabaseItem, + onResults: (results: Array<[string, ModeledMethod]>) => void | Promise, + progress: ProgressCallback, + token: CancellationToken, +) { + const generator = new FlowModelGenerator( + cli, + queryRunner, + queryStorageDir, + qlDir, + databaseItem, + progress, + token, + ); + + return generator.run(onResults); +} diff --git a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts index b8ff007c4..31bf3ef45 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts @@ -19,7 +19,7 @@ function readRowToMethod(row: any[]): string { return `${row[0]}.${row[1]}#${row[3]}${row[4]}`; } -const definitions: Record< +export const definitions: Record< Exclude, DataExtensionDefinition > = { diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index db25f3195..1adb70751 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -868,6 +868,7 @@ async function activateWithInstalledDistribution( await ensureDir(dataExtensionsEditorQueryStorageDir); const dataExtensionsEditorModule = new DataExtensionsEditorModule( ctx, + app, dbm, cliServer, qs, diff --git a/extensions/ql-vscode/src/pure/interface-types.ts b/extensions/ql-vscode/src/pure/interface-types.ts index b295695bd..de4828af2 100644 --- a/extensions/ql-vscode/src/pure/interface-types.ts +++ b/extensions/ql-vscode/src/pure/interface-types.ts @@ -15,6 +15,7 @@ import { import { RepositoriesFilterSortStateWithIds } from "./variant-analysis-filter-sort"; import { ErrorLike } from "./errors"; import { DataFlowPaths } from "../variant-analysis/shared/data-flow-paths"; +import { ModeledMethod } from "../data-extensions-editor/interface"; /** * This module contains types and code that are shared between @@ -497,16 +498,27 @@ export interface SetExistingYamlDataMessage { data: any; } +export interface AddModeledMethodsMessage { + t: "addModeledMethods"; + modeledMethods: Record; +} + export interface ApplyDataExtensionYamlMessage { t: "applyDataExtensionYaml"; yaml: string; } +export interface GenerateExternalApiMessage { + t: "generateExternalApi"; +} + export type ToDataExtensionsEditorMessage = | SetExternalApiResultsMessage | ShowProgressMessage - | SetExistingYamlDataMessage; + | SetExistingYamlDataMessage + | AddModeledMethodsMessage; export type FromDataExtensionsEditorMessage = | ViewLoadedMsg - | ApplyDataExtensionYamlMessage; + | ApplyDataExtensionYamlMessage + | GenerateExternalApiMessage; diff --git a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx index 86ae620fb..a241660d7 100644 --- a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx +++ b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx @@ -74,6 +74,20 @@ export function DataExtensionsEditor(): JSX.Element { }; }); + break; + case "addModeledMethods": + setModeledMethods((oldModeledMethods) => { + const filteredOldModeledMethods = Object.fromEntries( + Object.entries(oldModeledMethods).filter( + ([, value]) => value.type !== "none", + ), + ); + + return { + ...msg.modeledMethods, + ...filteredOldModeledMethods, + }; + }); break; default: assertNever(msg); @@ -168,6 +182,12 @@ export function DataExtensionsEditor(): JSX.Element { }); }, [methods, modeledMethods]); + const onGenerateClick = useCallback(() => { + vscode.postMessage({ + t: "generateExternalApi", + }); + }, []); + return ( {progress.maxStep > 0 && ( @@ -189,6 +209,12 @@ export function DataExtensionsEditor(): JSX.Element {

External API modelling

Apply +   + + Download and generate + +
+
From c245f338f5d0c0904b80537c90efdf7e241d1f71 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 5 Apr 2023 12:36:44 +0200 Subject: [PATCH 02/25] Extract external API usage query to separate file --- .../data-extensions-editor-view.ts | 112 ++++-------------- .../external-api-usage-query.ts | 102 ++++++++++++++++ 2 files changed, 122 insertions(+), 92 deletions(-) create mode 100644 extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 4c3819a63..e10716a2b 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -5,21 +5,14 @@ import { ToDataExtensionsEditorMessage, } from "../pure/interface-types"; import { ProgressUpdate } from "../progress"; -import { extLogger, TeeLogger } from "../common"; -import { CoreCompletedQuery, QueryRunner } from "../queryRunner"; -import { qlpackOfDatabase } from "../contextual/queryResolver"; -import { file } from "tmp-promise"; -import { writeFile } from "fs-extra"; -import { dump } from "js-yaml"; -import { - getOnDiskWorkspaceFolders, - showAndLogExceptionWithTelemetry, -} from "../helpers"; +import { QueryRunner } from "../queryRunner"; +import { showAndLogExceptionWithTelemetry } from "../helpers"; import { DatabaseItem } from "../local-databases"; import { CodeQLCliServer } from "../cli"; import { decodeBqrsToExternalApiUsages } from "./bqrs"; import { redactableError } from "../pure/errors"; import { asError, getErrorMessage } from "../pure/helpers-pure"; +import { getResults, runQuery } from "./external-api-usage-query"; export class DataExtensionsEditorView extends AbstractWebview< ToDataExtensionsEditorMessage, @@ -76,22 +69,34 @@ export class DataExtensionsEditorView extends AbstractWebview< } protected async loadExternalApiUsages(): Promise { + const cancellationTokenSource = new CancellationTokenSource(); + try { - const queryResult = await this.runQuery(); + const queryResult = await runQuery({ + cliServer: this.cliServer, + queryRunner: this.queryRunner, + databaseItem: this.databaseItem, + queryStorageDir: this.queryStorageDir, + progress: (progressUpdate: ProgressUpdate) => { + void this.showProgress(progressUpdate, 1500); + }, + token: cancellationTokenSource.token, + }); if (!queryResult) { await this.clearProgress(); return; } await this.showProgress({ - message: "Loading results", + message: "Decoding results", step: 1100, maxStep: 1500, }); - const bqrsPath = queryResult.outputDir.bqrsPath; - - const bqrsChunk = await this.getResults(bqrsPath); + const bqrsChunk = await getResults({ + cliServer: this.cliServer, + bqrsPath: queryResult.outputDir.bqrsPath, + }); if (!bqrsChunk) { await this.clearProgress(); return; @@ -120,83 +125,6 @@ export class DataExtensionsEditorView extends AbstractWebview< } } - private async runQuery(): Promise { - const qlpacks = await qlpackOfDatabase(this.cliServer, this.databaseItem); - - const packsToSearch = [qlpacks.dbschemePack]; - if (qlpacks.queryPack) { - packsToSearch.push(qlpacks.queryPack); - } - - const suiteFile = ( - await file({ - postfix: ".qls", - }) - ).path; - const suiteYaml = []; - for (const qlpack of packsToSearch) { - suiteYaml.push({ - from: qlpack, - queries: ".", - include: { - id: `${this.databaseItem.language}/telemetry/fetch-external-apis`, - }, - }); - } - await writeFile(suiteFile, dump(suiteYaml), "utf8"); - - const queries = await this.cliServer.resolveQueriesInSuite( - suiteFile, - getOnDiskWorkspaceFolders(), - ); - - if (queries.length !== 1) { - void extLogger.log(`Expected exactly one query, got ${queries.length}`); - return; - } - - const query = queries[0]; - - const tokenSource = new CancellationTokenSource(); - - const queryRun = this.queryRunner.createQueryRun( - this.databaseItem.databaseUri.fsPath, - { queryPath: query, quickEvalPosition: undefined }, - false, - getOnDiskWorkspaceFolders(), - undefined, - this.queryStorageDir, - undefined, - undefined, - ); - - return queryRun.evaluate( - (update) => this.showProgress(update, 1500), - tokenSource.token, - new TeeLogger(this.queryRunner.logger, queryRun.outputDir.logPath), - ); - } - - private async getResults(bqrsPath: string) { - const bqrsInfo = await this.cliServer.bqrsInfo(bqrsPath); - if (bqrsInfo["result-sets"].length !== 1) { - void extLogger.log( - `Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`, - ); - return undefined; - } - - const resultSet = bqrsInfo["result-sets"][0]; - - await this.showProgress({ - message: "Decoding results", - step: 1200, - maxStep: 1500, - }); - - return this.cliServer.bqrsDecode(bqrsPath, resultSet.name); - } - private async showProgress(update: ProgressUpdate, maxStep?: number) { await this.postMessage({ t: "showProgress", diff --git a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts new file mode 100644 index 000000000..54a69f2b9 --- /dev/null +++ b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts @@ -0,0 +1,102 @@ +import { CoreCompletedQuery, QueryRunner } from "../queryRunner"; +import { qlpackOfDatabase } from "../contextual/queryResolver"; +import { file } from "tmp-promise"; +import { writeFile } from "fs-extra"; +import { dump } from "js-yaml"; +import { getOnDiskWorkspaceFolders } from "../helpers"; +import { extLogger, TeeLogger } from "../common"; +import { CancellationToken } from "vscode"; +import { CodeQLCliServer } from "../cli"; +import { DatabaseItem } from "../local-databases"; +import { ProgressCallback } from "../progress"; + +export type RunQueryOptions = { + cliServer: CodeQLCliServer; + queryRunner: QueryRunner; + databaseItem: DatabaseItem; + queryStorageDir: string; + + progress: ProgressCallback; + token: CancellationToken; +}; + +export async function runQuery({ + cliServer, + queryRunner, + databaseItem, + queryStorageDir, + progress, + token, +}: RunQueryOptions): Promise { + const qlpacks = await qlpackOfDatabase(cliServer, databaseItem); + + const packsToSearch = [qlpacks.dbschemePack]; + if (qlpacks.queryPack) { + packsToSearch.push(qlpacks.queryPack); + } + + const suiteFile = ( + await file({ + postfix: ".qls", + }) + ).path; + const suiteYaml = []; + for (const qlpack of packsToSearch) { + suiteYaml.push({ + from: qlpack, + queries: ".", + include: { + id: `${databaseItem.language}/telemetry/fetch-external-apis`, + }, + }); + } + await writeFile(suiteFile, dump(suiteYaml), "utf8"); + + const queries = await cliServer.resolveQueriesInSuite( + suiteFile, + getOnDiskWorkspaceFolders(), + ); + + if (queries.length !== 1) { + void extLogger.log(`Expected exactly one query, got ${queries.length}`); + return; + } + + const query = queries[0]; + + const queryRun = queryRunner.createQueryRun( + databaseItem.databaseUri.fsPath, + { queryPath: query, quickEvalPosition: undefined }, + false, + getOnDiskWorkspaceFolders(), + undefined, + queryStorageDir, + undefined, + undefined, + ); + + return queryRun.evaluate( + progress, + token, + new TeeLogger(queryRunner.logger, queryRun.outputDir.logPath), + ); +} + +export type GetResultsOptions = { + cliServer: CodeQLCliServer; + bqrsPath: string; +}; + +export async function getResults({ cliServer, bqrsPath }: GetResultsOptions) { + const bqrsInfo = await cliServer.bqrsInfo(bqrsPath); + if (bqrsInfo["result-sets"].length !== 1) { + void extLogger.log( + `Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`, + ); + return undefined; + } + + const resultSet = bqrsInfo["result-sets"][0]; + + return cliServer.bqrsDecode(bqrsPath, resultSet.name); +} From 5ce3b2235167f1a130ff1d4b13b0b6649533be27 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 5 Apr 2023 13:50:48 +0200 Subject: [PATCH 03/25] Add tests for external API query This adds tests for the external API query and retrieving of results. It does not use the "real" CLI integration, but instead mocks the CLI server and query runner. To make mocking easier and require less type casting, I've narrowed some of the arguments of some other functions. They now use `Pick` to only require the properties they need. --- .../ql-vscode/src/contextual/queryResolver.ts | 4 +- .../data-extensions-editor-view.ts | 3 + .../external-api-usage-query.ts | 27 ++- extensions/ql-vscode/src/helpers.ts | 2 +- .../external-api-usage-query.test.ts | 222 ++++++++++++++++++ 5 files changed, 246 insertions(+), 12 deletions(-) create mode 100644 extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts diff --git a/extensions/ql-vscode/src/contextual/queryResolver.ts b/extensions/ql-vscode/src/contextual/queryResolver.ts index 7738971d0..03882465e 100644 --- a/extensions/ql-vscode/src/contextual/queryResolver.ts +++ b/extensions/ql-vscode/src/contextual/queryResolver.ts @@ -21,8 +21,8 @@ import { redactableError } from "../pure/errors"; import { QLPACK_FILENAMES } from "../pure/ql"; export async function qlpackOfDatabase( - cli: CodeQLCliServer, - db: DatabaseItem, + cli: Pick, + db: Pick, ): Promise { if (db.contents === undefined) { throw new Error("Database is invalid and cannot infer QLPack."); diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index e10716a2b..87224a70e 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -13,6 +13,7 @@ import { decodeBqrsToExternalApiUsages } from "./bqrs"; import { redactableError } from "../pure/errors"; import { asError, getErrorMessage } from "../pure/helpers-pure"; import { getResults, runQuery } from "./external-api-usage-query"; +import { extLogger } from "../common"; export class DataExtensionsEditorView extends AbstractWebview< ToDataExtensionsEditorMessage, @@ -77,6 +78,7 @@ export class DataExtensionsEditorView extends AbstractWebview< queryRunner: this.queryRunner, databaseItem: this.databaseItem, queryStorageDir: this.queryStorageDir, + logger: extLogger, progress: (progressUpdate: ProgressUpdate) => { void this.showProgress(progressUpdate, 1500); }, @@ -96,6 +98,7 @@ export class DataExtensionsEditorView extends AbstractWebview< const bqrsChunk = await getResults({ cliServer: this.cliServer, bqrsPath: queryResult.outputDir.bqrsPath, + logger: extLogger, }); if (!bqrsChunk) { await this.clearProgress(); diff --git a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts index 54a69f2b9..1535274a9 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts @@ -4,17 +4,18 @@ import { file } from "tmp-promise"; import { writeFile } from "fs-extra"; import { dump } from "js-yaml"; import { getOnDiskWorkspaceFolders } from "../helpers"; -import { extLogger, TeeLogger } from "../common"; +import { Logger, TeeLogger } from "../common"; import { CancellationToken } from "vscode"; import { CodeQLCliServer } from "../cli"; import { DatabaseItem } from "../local-databases"; import { ProgressCallback } from "../progress"; export type RunQueryOptions = { - cliServer: CodeQLCliServer; - queryRunner: QueryRunner; - databaseItem: DatabaseItem; + cliServer: Pick; + queryRunner: Pick; + databaseItem: Pick; queryStorageDir: string; + logger: Logger; progress: ProgressCallback; token: CancellationToken; @@ -25,6 +26,7 @@ export async function runQuery({ queryRunner, databaseItem, queryStorageDir, + logger, progress, token, }: RunQueryOptions): Promise { @@ -58,7 +60,7 @@ export async function runQuery({ ); if (queries.length !== 1) { - void extLogger.log(`Expected exactly one query, got ${queries.length}`); + void logger.log(`Expected exactly one query, got ${queries.length}`); return; } @@ -83,14 +85,19 @@ export async function runQuery({ } export type GetResultsOptions = { - cliServer: CodeQLCliServer; + cliServer: Pick; bqrsPath: string; + logger: Logger; }; -export async function getResults({ cliServer, bqrsPath }: GetResultsOptions) { +export async function getResults({ + cliServer, + bqrsPath, + logger, +}: GetResultsOptions) { const bqrsInfo = await cliServer.bqrsInfo(bqrsPath); if (bqrsInfo["result-sets"].length !== 1) { - void extLogger.log( + void logger.log( `Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`, ); return undefined; @@ -98,5 +105,7 @@ export async function getResults({ cliServer, bqrsPath }: GetResultsOptions) { const resultSet = bqrsInfo["result-sets"][0]; - return cliServer.bqrsDecode(bqrsPath, resultSet.name); + const result = await cliServer.bqrsDecode(bqrsPath, resultSet.name); + void logger.log(JSON.stringify(result)); + return result; } diff --git a/extensions/ql-vscode/src/helpers.ts b/extensions/ql-vscode/src/helpers.ts index 62f8d0d8d..c8d1839e0 100644 --- a/extensions/ql-vscode/src/helpers.ts +++ b/extensions/ql-vscode/src/helpers.ts @@ -478,7 +478,7 @@ function findStandardQueryPack( } export async function getQlPackForDbscheme( - cliServer: CodeQLCliServer, + cliServer: Pick, dbschemePath: string, ): Promise { const qlpacks = await cliServer.resolveQlpacks(getOnDiskWorkspaceFolders()); diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts new file mode 100644 index 000000000..9d76be823 --- /dev/null +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts @@ -0,0 +1,222 @@ +import { + getResults, + runQuery, +} from "../../../../src/data-extensions-editor/external-api-usage-query"; +import { createMockLogger } from "../../../__mocks__/loggerMock"; +import type { Uri } from "vscode"; +import { DatabaseKind } from "../../../../src/local-databases"; +import * as queryResolver from "../../../../src/contextual/queryResolver"; +import { file } from "tmp-promise"; +import { QueryResultType } from "../../../../src/pure/new-messages"; +import { readFile } from "fs-extra"; +import { load } from "js-yaml"; + +function createMockUri(path = "/a/b/c/foo"): Uri { + return { + scheme: "file", + authority: "", + path, + query: "", + fragment: "", + fsPath: path, + with: jest.fn(), + toJSON: jest.fn(), + }; +} + +describe("runQuery", () => { + it("runs the query", async () => { + jest.spyOn(queryResolver, "qlpackOfDatabase").mockResolvedValue({ + dbschemePack: "codeql/java-all", + dbschemePackIsLibraryPack: false, + queryPack: "codeql/java-queries", + }); + + const logPath = (await file()).path; + + const options = { + cliServer: { + resolveQlpacks: jest + .fn() + .mockRejectedValue( + new Error("Did not expect mocked method to be called"), + ), + resolveQueriesInSuite: jest + .fn() + .mockResolvedValue([ + "/home/github/codeql/java/ql/src/Telemetry/FetchExternalAPIs.ql", + ]), + }, + queryRunner: { + createQueryRun: jest.fn().mockReturnValue({ + evaluate: jest.fn().mockResolvedValue({ + resultType: QueryResultType.SUCCESS, + }), + outputDir: { + logPath, + }, + }), + logger: createMockLogger(), + }, + logger: createMockLogger(), + databaseItem: { + databaseUri: createMockUri("/a/b/c/src.zip"), + contents: { + kind: DatabaseKind.Database, + name: "foo", + datasetUri: createMockUri(), + }, + language: "java", + }, + queryStorageDir: "/tmp/queries", + progress: jest.fn(), + token: { + isCancellationRequested: false, + onCancellationRequested: jest.fn(), + }, + }; + const result = await runQuery(options); + + expect(result?.resultType).toEqual(QueryResultType.SUCCESS); + + expect(options.cliServer.resolveQueriesInSuite).toHaveBeenCalledWith( + expect.anything(), + [], + ); + const suiteFile = options.cliServer.resolveQueriesInSuite.mock.calls[0][0]; + const suiteFileContents = await readFile(suiteFile, "utf8"); + const suiteYaml = load(suiteFileContents); + expect(suiteYaml).toEqual([ + { + from: "codeql/java-all", + queries: ".", + include: { + id: "java/telemetry/fetch-external-apis", + }, + }, + { + from: "codeql/java-queries", + queries: ".", + include: { + id: "java/telemetry/fetch-external-apis", + }, + }, + ]); + + expect(options.queryRunner.createQueryRun).toHaveBeenCalledWith( + "/a/b/c/src.zip", + { + queryPath: + "/home/github/codeql/java/ql/src/Telemetry/FetchExternalAPIs.ql", + quickEvalPosition: undefined, + }, + false, + [], + undefined, + "/tmp/queries", + undefined, + undefined, + ); + }); +}); + +describe("getResults", () => { + const options = { + cliServer: { + bqrsInfo: jest.fn(), + bqrsDecode: jest.fn(), + }, + bqrsPath: "/tmp/results.bqrs", + logger: createMockLogger(), + }; + + it("returns undefined when there are no results", async () => { + options.cliServer.bqrsInfo.mockResolvedValue({ + "result-sets": [], + }); + + expect(await getResults(options)).toBeUndefined(); + expect(options.logger.log).toHaveBeenCalledWith( + expect.stringMatching(/Expected exactly one result set/), + ); + }); + + it("returns undefined when there are multiple result sets", async () => { + options.cliServer.bqrsInfo.mockResolvedValue({ + "result-sets": [ + { + name: "#select", + rows: 10, + columns: [ + { name: "apiName", kind: "s" }, + { name: "supported", kind: "b" }, + { name: "usage", kind: "e" }, + ], + }, + { + name: "#select2", + rows: 10, + columns: [ + { name: "apiName", kind: "s" }, + { name: "supported", kind: "b" }, + { name: "usage", kind: "e" }, + ], + }, + ], + }); + + expect(await getResults(options)).toBeUndefined(); + expect(options.logger.log).toHaveBeenCalledWith( + expect.stringMatching(/Expected exactly one result set/), + ); + }); + + it("gets the result set", async () => { + options.cliServer.bqrsInfo.mockResolvedValue({ + "result-sets": [ + { + name: "#select", + rows: 10, + columns: [ + { name: "apiName", kind: "s" }, + { name: "supported", kind: "b" }, + { name: "usage", kind: "e" }, + ], + }, + ], + "compatible-query-kinds": ["Table", "Tree", "Graph"], + }); + const decodedResultSet = { + columns: [ + { name: "apiName", kind: "String" }, + { name: "supported", kind: "Boolean" }, + { name: "usage", kind: "Entity" }, + ], + tuples: [ + [ + "java.io.PrintStream#println(String)", + true, + { + label: "println(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 29, + startColumn: 9, + endLine: 29, + endColumn: 49, + }, + }, + ], + ], + }; + options.cliServer.bqrsDecode.mockResolvedValue(decodedResultSet); + + const result = await getResults(options); + expect(result).toEqual(decodedResultSet); + expect(options.cliServer.bqrsInfo).toHaveBeenCalledWith(options.bqrsPath); + expect(options.cliServer.bqrsDecode).toHaveBeenCalledWith( + options.bqrsPath, + "#select", + ); + }); +}); From a1809b0bac6b1ab17f494f3c39220af26c077f1e Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 11:22:52 +0200 Subject: [PATCH 04/25] Merge similar messages for adding modeled methods --- .../data-extensions-editor-view.ts | 5 +++-- .../ql-vscode/src/pure/interface-types.ts | 14 +++++++------ .../DataExtensionsEditor.tsx | 21 +++++++------------ 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 1c4e04082..b3a022370 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -142,8 +142,8 @@ export class DataExtensionsEditorView extends AbstractWebview< } await this.postMessage({ - t: "setExistingModeledMethods", - existingModeledMethods, + t: "addModeledMethods", + modeledMethods: existingModeledMethods, }); } catch (e: unknown) { void extLogger.log(`Unable to read data extension YAML: ${e}`); @@ -250,6 +250,7 @@ export class DataExtensionsEditorView extends AbstractWebview< await this.postMessage({ t: "addModeledMethods", modeledMethods: modeledMethodsByName, + overrideNone: true, }); }, (update) => this.showProgress(update), diff --git a/extensions/ql-vscode/src/pure/interface-types.ts b/extensions/ql-vscode/src/pure/interface-types.ts index bc5599cf0..06ae988c6 100644 --- a/extensions/ql-vscode/src/pure/interface-types.ts +++ b/extensions/ql-vscode/src/pure/interface-types.ts @@ -493,14 +493,17 @@ export interface ShowProgressMessage { message: string; } -export interface SetExistingModeledMethods { - t: "setExistingModeledMethods"; - existingModeledMethods: Record; -} - export interface AddModeledMethodsMessage { t: "addModeledMethods"; modeledMethods: Record; + + /** + * If true, then any existing modeled methods set to "none" will be + * overwritten by the new modeled methods. Otherwise, the "none" modeled + * methods will not be overwritten, even if the new modeled methods + * contain a better model. + */ + overrideNone?: boolean; } export interface SaveModeledMethods { @@ -516,7 +519,6 @@ export interface GenerateExternalApiMessage { export type ToDataExtensionsEditorMessage = | SetExternalApiUsagesMessage | ShowProgressMessage - | SetExistingModeledMethods | AddModeledMethodsMessage; export type FromDataExtensionsEditorMessage = diff --git a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx index d757c6b89..594d4f898 100644 --- a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx +++ b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx @@ -56,23 +56,16 @@ export function DataExtensionsEditor(): JSX.Element { break; case "showProgress": setProgress(msg); - break; - case "setExistingModeledMethods": - setModeledMethods((oldModeledMethods) => { - return { - ...msg.existingModeledMethods, - ...oldModeledMethods, - }; - }); - break; case "addModeledMethods": setModeledMethods((oldModeledMethods) => { - const filteredOldModeledMethods = Object.fromEntries( - Object.entries(oldModeledMethods).filter( - ([, value]) => value.type !== "none", - ), - ); + const filteredOldModeledMethods = msg.overrideNone + ? Object.fromEntries( + Object.entries(oldModeledMethods).filter( + ([, value]) => value.type !== "none", + ), + ) + : oldModeledMethods; return { ...msg.modeledMethods, From 3ae16d95340985918644d937326e669885e017f6 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Mon, 3 Apr 2023 14:46:30 +0200 Subject: [PATCH 05/25] Add story for MethodRow --- .../MethodRow.stories.tsx | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 extensions/ql-vscode/src/stories/data-extensions-editor/MethodRow.stories.tsx diff --git a/extensions/ql-vscode/src/stories/data-extensions-editor/MethodRow.stories.tsx b/extensions/ql-vscode/src/stories/data-extensions-editor/MethodRow.stories.tsx new file mode 100644 index 000000000..9b87dc358 --- /dev/null +++ b/extensions/ql-vscode/src/stories/data-extensions-editor/MethodRow.stories.tsx @@ -0,0 +1,54 @@ +import * as React from "react"; + +import { ComponentMeta, ComponentStory } from "@storybook/react"; + +import { MethodRow as MethodRowComponent } from "../../view/data-extensions-editor/MethodRow"; + +export default { + title: "Data Extensions Editor/Method Row", + component: MethodRowComponent, +} as ComponentMeta; + +const Template: ComponentStory = (args) => ( + +); + +export const MethodRow = Template.bind({}); +MethodRow.args = { + externalApiUsage: { + signature: "org.sql2o.Sql2o#open()", + packageName: "org.sql2o", + typeName: "Sql2o", + methodName: "open", + methodParameters: "()", + supported: true, + usages: [ + { + label: "open(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 14, + startColumn: 24, + endLine: 14, + endColumn: 35, + }, + }, + { + label: "open(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 25, + startColumn: 24, + endLine: 25, + endColumn: 35, + }, + }, + ], + }, + modeledMethod: { + type: "summary", + input: "Argument[-1]", + output: "ReturnValue", + kind: "taint", + }, +}; From 1bcc13af7c97b8549438a5168effe8ffec0e8f88 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Mon, 3 Apr 2023 16:30:37 +0200 Subject: [PATCH 06/25] Add story for DataExtensionsEditor --- .../DataExtensionsEditor.stories.tsx | 223 ++++++++++++++++++ .../DataExtensionsEditor.tsx | 14 +- 2 files changed, 234 insertions(+), 3 deletions(-) create mode 100644 extensions/ql-vscode/src/stories/data-extensions-editor/DataExtensionsEditor.stories.tsx diff --git a/extensions/ql-vscode/src/stories/data-extensions-editor/DataExtensionsEditor.stories.tsx b/extensions/ql-vscode/src/stories/data-extensions-editor/DataExtensionsEditor.stories.tsx new file mode 100644 index 000000000..2964827ba --- /dev/null +++ b/extensions/ql-vscode/src/stories/data-extensions-editor/DataExtensionsEditor.stories.tsx @@ -0,0 +1,223 @@ +import * as React from "react"; + +import { ComponentMeta, ComponentStory } from "@storybook/react"; + +import { DataExtensionsEditor as DataExtensionsEditorComponent } from "../../view/data-extensions-editor/DataExtensionsEditor"; + +export default { + title: "Data Extensions Editor/Data Extensions Editor", + component: DataExtensionsEditorComponent, +} as ComponentMeta; + +const Template: ComponentStory = ( + args, +) => ; + +export const DataExtensionsEditor = Template.bind({}); +DataExtensionsEditor.args = { + initialExternalApiUsages: [ + { + signature: "org.sql2o.Connection#createQuery(String)", + packageName: "org.sql2o", + typeName: "Connection", + methodName: "createQuery", + methodParameters: "(String)", + supported: true, + usages: [ + { + label: "createQuery(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 15, + startColumn: 13, + endLine: 15, + endColumn: 56, + }, + }, + { + label: "createQuery(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 26, + startColumn: 13, + endLine: 26, + endColumn: 39, + }, + }, + ], + }, + { + signature: "org.sql2o.Query#executeScalar(Class)", + packageName: "org.sql2o", + typeName: "Query", + methodName: "executeScalar", + methodParameters: "(Class)", + supported: true, + usages: [ + { + label: "executeScalar(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 15, + startColumn: 13, + endLine: 15, + endColumn: 85, + }, + }, + { + label: "executeScalar(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 26, + startColumn: 13, + endLine: 26, + endColumn: 68, + }, + }, + ], + }, + { + signature: "org.sql2o.Sql2o#open()", + packageName: "org.sql2o", + typeName: "Sql2o", + methodName: "open", + methodParameters: "()", + supported: false, + usages: [ + { + label: "open(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 14, + startColumn: 24, + endLine: 14, + endColumn: 35, + }, + }, + { + label: "open(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 25, + startColumn: 24, + endLine: 25, + endColumn: 35, + }, + }, + ], + }, + { + signature: "java.io.PrintStream#println(String)", + packageName: "java.io", + typeName: "PrintStream", + methodName: "println", + methodParameters: "(String)", + supported: true, + usages: [ + { + label: "println(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 29, + startColumn: 9, + endLine: 29, + endColumn: 49, + }, + }, + ], + }, + { + signature: + "org.springframework.boot.SpringApplication#run(Class,String[])", + packageName: "org.springframework.boot", + typeName: "SpringApplication", + methodName: "run", + methodParameters: "(Class,String[])", + supported: false, + usages: [ + { + label: "run(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/Sql2oExampleApplication.java", + startLine: 9, + startColumn: 9, + endLine: 9, + endColumn: 66, + }, + }, + ], + }, + { + signature: "org.sql2o.Sql2o#Sql2o(String,String,String)", + packageName: "org.sql2o", + typeName: "Sql2o", + methodName: "Sql2o", + methodParameters: "(String,String,String)", + supported: false, + usages: [ + { + label: "new Sql2o(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 10, + startColumn: 33, + endLine: 10, + endColumn: 88, + }, + }, + ], + }, + { + signature: "org.sql2o.Sql2o#Sql2o(String)", + packageName: "org.sql2o", + typeName: "Sql2o", + methodName: "Sql2o", + methodParameters: "(String)", + supported: false, + usages: [ + { + label: "new Sql2o(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 23, + startColumn: 23, + endLine: 23, + endColumn: 36, + }, + }, + ], + }, + ], + initialModeledMethods: { + "org.sql2o.Sql2o#Sql2o(String)": { + type: "sink", + input: "Argument[0]", + output: "", + kind: "jndi-injection", + }, + "org.sql2o.Connection#createQuery(String)": { + type: "summary", + input: "Argument[-1]", + output: "ReturnValue", + kind: "taint", + }, + "org.sql2o.Sql2o#open()": { + type: "summary", + input: "Argument[-1]", + output: "ReturnValue", + kind: "taint", + }, + "org.sql2o.Query#executeScalar(Class)": { + type: "neutral", + input: "", + output: "", + kind: "", + }, + "org.sql2o.Sql2o#Sql2o(String,String,String)": { + type: "neutral", + input: "", + output: "", + kind: "", + }, + }, +}; diff --git a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx index 97c6d8994..2e42e6697 100644 --- a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx +++ b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx @@ -33,13 +33,21 @@ const ProgressBar = styled.div` background-color: var(--vscode-progressBar-background); `; -export function DataExtensionsEditor(): JSX.Element { +type Props = { + initialExternalApiUsages?: ExternalApiUsage[]; + initialModeledMethods?: Record; +}; + +export function DataExtensionsEditor({ + initialExternalApiUsages = [], + initialModeledMethods = {}, +}: Props): JSX.Element { const [externalApiUsages, setExternalApiUsages] = useState< ExternalApiUsage[] - >([]); + >(initialExternalApiUsages); const [modeledMethods, setModeledMethods] = useState< Record - >({}); + >(initialModeledMethods); const [progress, setProgress] = useState>({ step: 0, maxStep: 0, From bb05220b2c5acd906d9d61ba963aff17c3d9373b Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Mon, 3 Apr 2023 16:37:29 +0200 Subject: [PATCH 07/25] Fix Babel compilation error The default Storybook Babel config did not recognize the `public` keyword in our custom errors (e.g. `ExhaustivityCheckingError` and `RedactableError`). To fix this, we can use Storybook's V7 mode to supply a custom Babel config. This fixes the compilation error. See: https://storybook.js.org/docs/react/configure/babel --- extensions/ql-vscode/.babelrc.json | 16 ++++++++++++++++ extensions/ql-vscode/.storybook/main.ts | 3 +++ 2 files changed, 19 insertions(+) create mode 100644 extensions/ql-vscode/.babelrc.json diff --git a/extensions/ql-vscode/.babelrc.json b/extensions/ql-vscode/.babelrc.json new file mode 100644 index 000000000..00ca841a4 --- /dev/null +++ b/extensions/ql-vscode/.babelrc.json @@ -0,0 +1,16 @@ +{ + "sourceType": "unambiguous", + "presets": [ + [ + "@babel/preset-env", + { + "targets": { + "chrome": 100 + } + } + ], + "@babel/preset-typescript", + "@babel/preset-react" + ], + "plugins": [] +} diff --git a/extensions/ql-vscode/.storybook/main.ts b/extensions/ql-vscode/.storybook/main.ts index db997443a..43b16ab27 100644 --- a/extensions/ql-vscode/.storybook/main.ts +++ b/extensions/ql-vscode/.storybook/main.ts @@ -12,6 +12,9 @@ const config: StorybookConfig = { core: { builder: "@storybook/builder-webpack5", }, + features: { + babelModeV7: true, + }, }; module.exports = config; From 4fa229f8a2295c1ab5052d2148d55a8c54aa6327 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:16:37 +0200 Subject: [PATCH 08/25] Rename generateExternalApi to generateModeledMethods --- .../src/data-extensions-editor/data-extensions-editor-view.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index cccae294a..e0d05db85 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -95,7 +95,7 @@ export class DataExtensionsEditorView extends AbstractWebview< break; case "generateExternalApi": - await this.generateExternalApi(); + await this.generateModeledMethods(); break; default: @@ -222,7 +222,7 @@ export class DataExtensionsEditorView extends AbstractWebview< } } - protected async generateExternalApi(): Promise { + protected async generateModeledMethods(): Promise { const tokenSource = new CancellationTokenSource(); const selectedDatabase = this.databaseManager.currentDatabaseItem; From 42d40347442205590f548fe9a242dc266f09ac4f Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:20:01 +0200 Subject: [PATCH 09/25] Extract getting ql submodule folder to function --- .../data-extensions-editor-view.ts | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index e0d05db85..ec922f238 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -5,6 +5,7 @@ import { ViewColumn, window, workspace, + WorkspaceFolder, } from "vscode"; import { AbstractWebview, WebviewPanelConfig } from "../abstract-webview"; import { @@ -37,6 +38,19 @@ import { createDataExtensionYaml, loadDataExtensionYaml } from "./yaml"; import { ExternalApiUsage } from "./external-api-usage"; import { ModeledMethod } from "./modeled-method"; +function getQlSubmoduleFolder(): WorkspaceFolder | undefined { + const workspaceFolder = workspace.workspaceFolders?.find( + (folder) => folder.name === "ql", + ); + if (!workspaceFolder) { + void extLogger.log("No workspace folder 'ql' found"); + + return; + } + + return workspaceFolder; +} + export class DataExtensionsEditorView extends AbstractWebview< ToDataExtensionsEditorMessage, FromDataExtensionsEditorMessage @@ -245,12 +259,8 @@ export class DataExtensionsEditorView extends AbstractWebview< await this.databaseManager.setCurrentDatabaseItem(selectedDatabase); - const workspaceFolder = workspace.workspaceFolders?.find( - (folder) => folder.name === "ql", - ); + const workspaceFolder = getQlSubmoduleFolder(); if (!workspaceFolder) { - void extLogger.log("No workspace folder 'ql' found"); - return; } @@ -404,12 +414,8 @@ export class DataExtensionsEditorView extends AbstractWebview< } private calculateModelFilename(): string | undefined { - const workspaceFolder = workspace.workspaceFolders?.find( - (folder) => folder.name === "ql", - ); + const workspaceFolder = getQlSubmoduleFolder(); if (!workspaceFolder) { - void extLogger.log("No workspace folder 'ql' found"); - return; } From d3ff87ab71116fa196055593cc62a1862c9802ba Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:21:48 +0200 Subject: [PATCH 10/25] Update language used for generating modeled methods --- .../src/data-extensions-editor/data-extensions-editor-view.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index ec922f238..9a41ce59c 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -267,7 +267,7 @@ export class DataExtensionsEditorView extends AbstractWebview< await this.showProgress({ step: 0, maxStep: 4000, - message: "Generating external API", + message: "Generating modeled methods for library", }); try { From 5a3a1a5cd7633796201831dff45767a0c4e496a2 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:23:10 +0200 Subject: [PATCH 11/25] Add comment for flow model progress --- .../data-extensions-editor/data-extensions-editor-view.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 9a41ce59c..a4383b23a 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -395,6 +395,13 @@ export class DataExtensionsEditorView extends AbstractWebview< * that there's 1000 steps of the query progress since that takes the most time, and then * an additional 500 steps for the rest of the work. The progress doesn't need to be 100% * accurate, so this is just a rough estimate. + * + * For generating the modeled methods for an external library, the max step is 4000. This is + * based on the following steps: + * - 1000 for the summary model + * - 1000 for the sink model + * - 1000 for the source model + * - 1000 for the neutral model */ private async showProgress(update: ProgressUpdate, maxStep?: number) { await this.postMessage({ From 3214a376ad046d158216c6b31632b7253c8509b7 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:24:25 +0200 Subject: [PATCH 12/25] Clarify definitions to be extensible predicate definitions --- .../generate-flow-model.ts | 4 +-- .../src/data-extensions-editor/yaml.ts | 26 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index d13bf00fc..6eb71ab18 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -4,7 +4,7 @@ import { join } from "path"; import { QueryRunner } from "../queryRunner"; import { CodeQLCliServer } from "../cli"; import { extLogger, TeeLogger } from "../common"; -import { definitions } from "./yaml"; +import { extensiblePredicateDefinitions } from "./yaml"; import { ProgressCallback } from "../progress"; import { getOnDiskWorkspaceFolders } from "../helpers"; import { ModeledMethod, ModeledMethodType } from "./modeled-method"; @@ -25,7 +25,7 @@ class FlowModelGenerator { queryName: string, queryStep: number, ): Promise | undefined> { - const definition = definitions[type]; + const definition = extensiblePredicateDefinitions[type]; const query = join( this.qlDir, diff --git a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts index 7d72e434d..de7b60350 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts @@ -6,8 +6,8 @@ type ExternalApiUsageByType = { modeledMethod: ModeledMethod; }; -type DataExtensionDefinition = { - extensible: string; +type ExtensiblePredicateDefinition = { + extensiblePredicate: string; generateMethodDefinition: (method: ExternalApiUsageByType) => any[]; readModeledMethod: (row: any[]) => [string, ModeledMethod] | undefined; }; @@ -16,12 +16,12 @@ function readRowToMethod(row: any[]): string { return `${row[0]}.${row[1]}#${row[3]}${row[4]}`; } -export const definitions: Record< +export const extensiblePredicateDefinitions: Record< Exclude, - DataExtensionDefinition + ExtensiblePredicateDefinition > = { source: { - extensible: "sourceModel", + extensiblePredicate: "sourceModel", // extensible predicate sourceModel( // string package, string type, boolean subtypes, string name, string signature, string ext, // string output, string kind, string provenance @@ -48,7 +48,7 @@ export const definitions: Record< ], }, sink: { - extensible: "sinkModel", + extensiblePredicate: "sinkModel", // extensible predicate sinkModel( // string package, string type, boolean subtypes, string name, string signature, string ext, // string input, string kind, string provenance @@ -75,7 +75,7 @@ export const definitions: Record< ], }, summary: { - extensible: "summaryModel", + extensiblePredicate: "summaryModel", // extensible predicate summaryModel( // string package, string type, boolean subtypes, string name, string signature, string ext, // string input, string output, string kind, string provenance @@ -103,7 +103,7 @@ export const definitions: Record< ], }, neutral: { - extensible: "neutralModel", + extensiblePredicate: "neutralModel", // extensible predicate neutralModel( // string package, string type, string name, string signature, string provenance // ); @@ -128,7 +128,7 @@ export const definitions: Record< function createDataProperty( methods: ExternalApiUsageByType[], - definition: DataExtensionDefinition, + definition: ExtensiblePredicateDefinition, ) { if (methods.length === 0) { return " []"; @@ -169,10 +169,10 @@ export function createDataExtensionYaml( } } - const extensions = Object.entries(definitions).map( + const extensions = Object.entries(extensiblePredicateDefinitions).map( ([type, definition]) => ` - addsTo: pack: codeql/java-all - extensible: ${definition.extensible} + extensible: ${definition.extensiblePredicate} data:${createDataProperty( methodsByType[type as Exclude], definition, @@ -214,8 +214,8 @@ export function loadDataExtensionYaml( continue; } - const definition = Object.values(definitions).find( - (definition) => definition.extensible === extensible, + const definition = Object.values(extensiblePredicateDefinitions).find( + (definition) => definition.extensiblePredicate === extensible, ); if (!definition) { continue; From 0d00e5c5b7054bb9dfc4f53ae456a18984b8311e Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:27:34 +0200 Subject: [PATCH 13/25] Remove undefined from readModeledMethod return type --- .../data-extensions-editor/generate-flow-model.ts | 12 ++++-------- .../ql-vscode/src/data-extensions-editor/yaml.ts | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index 6eb71ab18..1ab535b69 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -72,15 +72,11 @@ class FlowModelGenerator { const results = decodedResults.tuples; - return results - .map((result): [string, ModeledMethod] | undefined => { - const row = result[0] as string; + return results.map((result): [string, ModeledMethod] => { + const row = result[0] as string; - return definition.readModeledMethod(row.split(";")); - }) - .filter( - (result): result is [string, ModeledMethod] => result !== undefined, - ); + return definition.readModeledMethod(row.split(";")); + }); } async run( diff --git a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts index de7b60350..c573d1f88 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts @@ -9,7 +9,7 @@ type ExternalApiUsageByType = { type ExtensiblePredicateDefinition = { extensiblePredicate: string; generateMethodDefinition: (method: ExternalApiUsageByType) => any[]; - readModeledMethod: (row: any[]) => [string, ModeledMethod] | undefined; + readModeledMethod: (row: any[]) => [string, ModeledMethod]; }; function readRowToMethod(row: any[]): string { From d5403ad92656b7f838753c424127c1d93ab2d6ca Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:33:43 +0200 Subject: [PATCH 14/25] Introduce type for modeled method with signature --- .../data-extensions-editor-view.ts | 2 +- .../generate-flow-model.ts | 15 ++++--- .../data-extensions-editor/modeled-method.ts | 5 +++ .../src/data-extensions-editor/yaml.ts | 44 ++++++++++--------- 4 files changed, 38 insertions(+), 28 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index a4383b23a..1622f6875 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -281,7 +281,7 @@ export class DataExtensionsEditorView extends AbstractWebview< const modeledMethodsByName: Record = {}; for (const result of results) { - modeledMethodsByName[result[0]] = result[1]; + modeledMethodsByName[result.signature] = result.modeledMethod; } await this.postMessage({ diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index 1ab535b69..8f737fe3e 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -7,7 +7,10 @@ import { extLogger, TeeLogger } from "../common"; import { extensiblePredicateDefinitions } from "./yaml"; import { ProgressCallback } from "../progress"; import { getOnDiskWorkspaceFolders } from "../helpers"; -import { ModeledMethod, ModeledMethodType } from "./modeled-method"; +import { + ModeledMethodType, + ModeledMethodWithSignature, +} from "./modeled-method"; class FlowModelGenerator { constructor( @@ -24,7 +27,7 @@ class FlowModelGenerator { type: Exclude, queryName: string, queryStep: number, - ): Promise | undefined> { + ): Promise { const definition = extensiblePredicateDefinitions[type]; const query = join( @@ -72,7 +75,7 @@ class FlowModelGenerator { const results = decodedResults.tuples; - return results.map((result): [string, ModeledMethod] => { + return results.map((result) => { const row = result[0] as string; return definition.readModeledMethod(row.split(";")); @@ -80,9 +83,7 @@ class FlowModelGenerator { } async run( - onResults: ( - results: Array<[string, ModeledMethod]>, - ) => void | Promise, + onResults: (results: ModeledMethodWithSignature[]) => void | Promise, ) { const summaryResults = await this.getAddsTo( "summary", @@ -124,7 +125,7 @@ export async function generateFlowModel( queryStorageDir: string, qlDir: string, databaseItem: DatabaseItem, - onResults: (results: Array<[string, ModeledMethod]>) => void | Promise, + onResults: (results: ModeledMethodWithSignature[]) => void | Promise, progress: ProgressCallback, token: CancellationToken, ) { diff --git a/extensions/ql-vscode/src/data-extensions-editor/modeled-method.ts b/extensions/ql-vscode/src/data-extensions-editor/modeled-method.ts index cac4fa466..27c3d5f15 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/modeled-method.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/modeled-method.ts @@ -11,3 +11,8 @@ export type ModeledMethod = { output: string; kind: string; }; + +export type ModeledMethodWithSignature = { + signature: string; + modeledMethod: ModeledMethod; +}; diff --git a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts index c573d1f88..95383173d 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts @@ -1,5 +1,9 @@ import { ExternalApiUsage } from "./external-api-usage"; -import { ModeledMethod, ModeledMethodType } from "./modeled-method"; +import { + ModeledMethod, + ModeledMethodType, + ModeledMethodWithSignature, +} from "./modeled-method"; type ExternalApiUsageByType = { externalApiUsage: ExternalApiUsage; @@ -9,7 +13,7 @@ type ExternalApiUsageByType = { type ExtensiblePredicateDefinition = { extensiblePredicate: string; generateMethodDefinition: (method: ExternalApiUsageByType) => any[]; - readModeledMethod: (row: any[]) => [string, ModeledMethod]; + readModeledMethod: (row: any[]) => ModeledMethodWithSignature; }; function readRowToMethod(row: any[]): string { @@ -37,15 +41,15 @@ export const extensiblePredicateDefinitions: Record< method.modeledMethod.kind, "manual", ], - readModeledMethod: (row) => [ - readRowToMethod(row), - { + readModeledMethod: (row) => ({ + signature: readRowToMethod(row), + modeledMethod: { type: "source", input: "", output: row[6], kind: row[7], }, - ], + }), }, sink: { extensiblePredicate: "sinkModel", @@ -64,15 +68,15 @@ export const extensiblePredicateDefinitions: Record< method.modeledMethod.kind, "manual", ], - readModeledMethod: (row) => [ - readRowToMethod(row), - { + readModeledMethod: (row) => ({ + signature: readRowToMethod(row), + modeledMethod: { type: "sink", input: row[6], output: "", kind: row[7], }, - ], + }), }, summary: { extensiblePredicate: "summaryModel", @@ -92,15 +96,15 @@ export const extensiblePredicateDefinitions: Record< method.modeledMethod.kind, "manual", ], - readModeledMethod: (row) => [ - readRowToMethod(row), - { + readModeledMethod: (row) => ({ + signature: readRowToMethod(row), + modeledMethod: { type: "summary", input: row[6], output: row[7], kind: row[8], }, - ], + }), }, neutral: { extensiblePredicate: "neutralModel", @@ -114,15 +118,15 @@ export const extensiblePredicateDefinitions: Record< method.externalApiUsage.methodParameters, "manual", ], - readModeledMethod: (row) => [ - `${row[0]}.${row[1]}#${row[2]}${row[3]}`, - { + readModeledMethod: (row) => ({ + signature: `${row[0]}.${row[1]}#${row[2]}${row[3]}`, + modeledMethod: { type: "neutral", input: "", output: "", kind: "", }, - ], + }), }, }; @@ -227,9 +231,9 @@ export function loadDataExtensionYaml( continue; } - const [apiInfo, modeledMethod] = result; + const { signature, modeledMethod } = result; - modeledMethods[apiInfo] = modeledMethod; + modeledMethods[signature] = modeledMethod; } } From 18db74ed2daa8102a637cd7c317913a24ac45f29 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:35:04 +0200 Subject: [PATCH 15/25] Add type check for query results --- .../data-extensions-editor/generate-flow-model.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index 8f737fe3e..445754d91 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -75,11 +75,16 @@ class FlowModelGenerator { const results = decodedResults.tuples; - return results.map((result) => { - const row = result[0] as string; + return ( + results + // This is just a sanity check. The query should only return strings. + .filter((result) => typeof result[0] === "string") + .map((result) => { + const row = result[0] as string; - return definition.readModeledMethod(row.split(";")); - }); + return definition.readModeledMethod(row.split(";")); + }) + ); } async run( From af8e0bb45478a7ba11750e4cb5bc30133bf06cc1 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:49:20 +0200 Subject: [PATCH 16/25] Rename getAddsTo --- .../data-extensions-editor/generate-flow-model.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index 445754d91..8c3c61013 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -23,7 +23,7 @@ class FlowModelGenerator { private readonly token: CancellationToken, ) {} - async getAddsTo( + private async getModeledMethodsFromFlow( type: Exclude, queryName: string, queryStep: number, @@ -90,7 +90,7 @@ class FlowModelGenerator { async run( onResults: (results: ModeledMethodWithSignature[]) => void | Promise, ) { - const summaryResults = await this.getAddsTo( + const summaryResults = await this.getModeledMethodsFromFlow( "summary", "CaptureSummaryModels.ql", 0, @@ -99,12 +99,16 @@ class FlowModelGenerator { await onResults(summaryResults); } - const sinkResults = await this.getAddsTo("sink", "CaptureSinkModels.ql", 1); + const sinkResults = await this.getModeledMethodsFromFlow( + "sink", + "CaptureSinkModels.ql", + 1, + ); if (sinkResults) { await onResults(sinkResults); } - const sourceResults = await this.getAddsTo( + const sourceResults = await this.getModeledMethodsFromFlow( "source", "CaptureSourceModels.ql", 2, @@ -113,7 +117,7 @@ class FlowModelGenerator { await onResults(sourceResults); } - const neutralResults = await this.getAddsTo( + const neutralResults = await this.getModeledMethodsFromFlow( "neutral", "CaptureNeutralModels.ql", 3, From dbc7f90c205c28cae22d74ebcb110e66063a23ff Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:54:20 +0200 Subject: [PATCH 17/25] Throw error instead of returning undefined --- .../data-extensions-editor/data-extensions-editor-view.ts | 6 +++++- .../src/data-extensions-editor/generate-flow-model.ts | 7 +++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 1622f6875..b4da79e8a 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -294,7 +294,11 @@ export class DataExtensionsEditorView extends AbstractWebview< tokenSource.token, ); } catch (e: unknown) { - void extLogger.log(`Error: ${getErrorMessage(e)}`); + void showAndLogExceptionWithTelemetry( + redactableError( + asError(e), + )`Failed to generate flow model: ${getErrorMessage(e)}`, + ); } await this.databaseManager.removeDatabaseItem( diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index 8c3c61013..e60960eb5 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -3,7 +3,7 @@ import { DatabaseItem } from "../local-databases"; import { join } from "path"; import { QueryRunner } from "../queryRunner"; import { CodeQLCliServer } from "../cli"; -import { extLogger, TeeLogger } from "../common"; +import { TeeLogger } from "../common"; import { extensiblePredicateDefinitions } from "./yaml"; import { ProgressCallback } from "../progress"; import { getOnDiskWorkspaceFolders } from "../helpers"; @@ -27,7 +27,7 @@ class FlowModelGenerator { type: Exclude, queryName: string, queryStep: number, - ): Promise { + ): Promise { const definition = extensiblePredicateDefinitions[type]; const query = join( @@ -63,10 +63,9 @@ class FlowModelGenerator { const bqrsInfo = await this.cli.bqrsInfo(bqrsPath); if (bqrsInfo["result-sets"].length !== 1) { - void extLogger.log( + throw new Error( `Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`, ); - return undefined; } const resultSet = bqrsInfo["result-sets"][0]; From 4e8df309fb5d231ccf3f0fed0fb04a6b442f2d1a Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Tue, 11 Apr 2023 10:22:10 +0200 Subject: [PATCH 18/25] Add comments for adding flow database --- .../data-extensions-editor/data-extensions-editor-view.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index b4da79e8a..5e7be0a45 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -241,6 +241,9 @@ export class DataExtensionsEditorView extends AbstractWebview< const selectedDatabase = this.databaseManager.currentDatabaseItem; + // The external API methods are in the library source code, so we need to ask + // the user to import the library database. We need to have the database + // imported to the query server, so we need to register it to our workspace. const database = await promptImportGithubDatabase( this.app.commands, this.databaseManager, @@ -257,6 +260,8 @@ export class DataExtensionsEditorView extends AbstractWebview< return; } + // The library database was set as the current database by importing it, + // but we need to set it back to the originally selected database. await this.databaseManager.setCurrentDatabaseItem(selectedDatabase); const workspaceFolder = getQlSubmoduleFolder(); @@ -301,6 +306,8 @@ export class DataExtensionsEditorView extends AbstractWebview< ); } + // After the flow model has been generated, we can remove the temporary database + // which we used for generating the flow model. await this.databaseManager.removeDatabaseItem( () => this.showProgress({ From 102976e167b804dd205629fc4c11439383fc68a4 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Tue, 11 Apr 2023 10:27:46 +0200 Subject: [PATCH 19/25] Use functions instead of class for generating flow model We were using a single-use class for generating the flow model, while we are actually able to do it using two functions. This is more in line with our existing codebase. --- .../data-extensions-editor-view.ts | 20 +- .../generate-flow-model.ts | 244 +++++++++--------- 2 files changed, 127 insertions(+), 137 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 5e7be0a45..7975c884b 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -276,13 +276,13 @@ export class DataExtensionsEditorView extends AbstractWebview< }); try { - await generateFlowModel( - this.cliServer, - this.queryRunner, - this.queryStorageDir, - workspaceFolder.uri.fsPath, - database, - async (results) => { + await generateFlowModel({ + cliServer: this.cliServer, + queryRunner: this.queryRunner, + queryStorageDir: this.queryStorageDir, + qlDir: workspaceFolder.uri.fsPath, + databaseItem: database, + onResults: async (results) => { const modeledMethodsByName: Record = {}; for (const result of results) { @@ -295,9 +295,9 @@ export class DataExtensionsEditorView extends AbstractWebview< overrideNone: true, }); }, - (update) => this.showProgress(update), - tokenSource.token, - ); + progress: (update) => this.showProgress(update), + token: tokenSource.token, + }); } catch (e: unknown) { void showAndLogExceptionWithTelemetry( redactableError( diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index e60960eb5..f6125d0f3 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -12,140 +12,130 @@ import { ModeledMethodWithSignature, } from "./modeled-method"; -class FlowModelGenerator { - constructor( - private readonly cli: CodeQLCliServer, - private readonly queryRunner: QueryRunner, - private readonly queryStorageDir: string, - private readonly qlDir: string, - private readonly databaseItem: DatabaseItem, - private readonly progress: ProgressCallback, - private readonly token: CancellationToken, - ) {} +type FlowModelOptions = { + cliServer: CodeQLCliServer; + queryRunner: QueryRunner; + queryStorageDir: string; + qlDir: string; + databaseItem: DatabaseItem; + progress: ProgressCallback; + token: CancellationToken; + onResults: (results: ModeledMethodWithSignature[]) => void | Promise; +}; - private async getModeledMethodsFromFlow( - type: Exclude, - queryName: string, - queryStep: number, - ): Promise { - const definition = extensiblePredicateDefinitions[type]; - - const query = join( - this.qlDir, - this.databaseItem.language, - "ql/src/utils/modelgenerator", - queryName, - ); - - const queryRun = this.queryRunner.createQueryRun( - this.databaseItem.databaseUri.fsPath, - { queryPath: query, quickEvalPosition: undefined }, - false, - getOnDiskWorkspaceFolders(), - undefined, - this.queryStorageDir, - undefined, - undefined, - ); - - const queryResult = await queryRun.evaluate( - ({ step, message }) => - this.progress({ - message: `Generating ${type} model: ${message}`, - step: queryStep * 1000 + step, - maxStep: 4000, - }), - this.token, - new TeeLogger(this.queryRunner.logger, queryRun.outputDir.logPath), - ); - - const bqrsPath = queryResult.outputDir.bqrsPath; - - const bqrsInfo = await this.cli.bqrsInfo(bqrsPath); - if (bqrsInfo["result-sets"].length !== 1) { - throw new Error( - `Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`, - ); - } - - const resultSet = bqrsInfo["result-sets"][0]; - - const decodedResults = await this.cli.bqrsDecode(bqrsPath, resultSet.name); - - const results = decodedResults.tuples; - - return ( - results - // This is just a sanity check. The query should only return strings. - .filter((result) => typeof result[0] === "string") - .map((result) => { - const row = result[0] as string; - - return definition.readModeledMethod(row.split(";")); - }) - ); - } - - async run( - onResults: (results: ModeledMethodWithSignature[]) => void | Promise, - ) { - const summaryResults = await this.getModeledMethodsFromFlow( - "summary", - "CaptureSummaryModels.ql", - 0, - ); - if (summaryResults) { - await onResults(summaryResults); - } - - const sinkResults = await this.getModeledMethodsFromFlow( - "sink", - "CaptureSinkModels.ql", - 1, - ); - if (sinkResults) { - await onResults(sinkResults); - } - - const sourceResults = await this.getModeledMethodsFromFlow( - "source", - "CaptureSourceModels.ql", - 2, - ); - if (sourceResults) { - await onResults(sourceResults); - } - - const neutralResults = await this.getModeledMethodsFromFlow( - "neutral", - "CaptureNeutralModels.ql", - 3, - ); - if (neutralResults) { - await onResults(neutralResults); - } - } -} - -export async function generateFlowModel( - cli: CodeQLCliServer, - queryRunner: QueryRunner, - queryStorageDir: string, - qlDir: string, - databaseItem: DatabaseItem, - onResults: (results: ModeledMethodWithSignature[]) => void | Promise, - progress: ProgressCallback, - token: CancellationToken, -) { - const generator = new FlowModelGenerator( - cli, +async function getModeledMethodsFromFlow( + type: Exclude, + queryName: string, + queryStep: number, + { + cliServer, queryRunner, queryStorageDir, qlDir, databaseItem, progress, token, + }: Omit, +): Promise { + const definition = extensiblePredicateDefinitions[type]; + + const query = join( + qlDir, + databaseItem.language, + "ql/src/utils/modelgenerator", + queryName, ); - return generator.run(onResults); + const queryRun = queryRunner.createQueryRun( + databaseItem.databaseUri.fsPath, + { queryPath: query, quickEvalPosition: undefined }, + false, + getOnDiskWorkspaceFolders(), + undefined, + queryStorageDir, + undefined, + undefined, + ); + + const queryResult = await queryRun.evaluate( + ({ step, message }) => + progress({ + message: `Generating ${type} model: ${message}`, + step: queryStep * 1000 + step, + maxStep: 4000, + }), + token, + new TeeLogger(queryRunner.logger, queryRun.outputDir.logPath), + ); + + const bqrsPath = queryResult.outputDir.bqrsPath; + + const bqrsInfo = await cliServer.bqrsInfo(bqrsPath); + if (bqrsInfo["result-sets"].length !== 1) { + throw new Error( + `Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`, + ); + } + + const resultSet = bqrsInfo["result-sets"][0]; + + const decodedResults = await cliServer.bqrsDecode(bqrsPath, resultSet.name); + + const results = decodedResults.tuples; + + return ( + results + // This is just a sanity check. The query should only return strings. + .filter((result) => typeof result[0] === "string") + .map((result) => { + const row = result[0] as string; + + return definition.readModeledMethod(row.split(";")); + }) + ); +} + +export async function generateFlowModel({ + onResults, + ...options +}: FlowModelOptions) { + const summaryResults = await getModeledMethodsFromFlow( + "summary", + "CaptureSummaryModels.ql", + 0, + options, + ); + if (summaryResults) { + await onResults(summaryResults); + } + + const sinkResults = await getModeledMethodsFromFlow( + "sink", + "CaptureSinkModels.ql", + 1, + options, + ); + if (sinkResults) { + await onResults(sinkResults); + } + + const sourceResults = await getModeledMethodsFromFlow( + "source", + "CaptureSourceModels.ql", + 2, + options, + ); + if (sourceResults) { + await onResults(sourceResults); + } + + const neutralResults = await getModeledMethodsFromFlow( + "neutral", + "CaptureNeutralModels.ql", + 3, + options, + ); + if (neutralResults) { + await onResults(neutralResults); + } } From 41e0dc2961607347005e51707e752ef490c0c684 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Tue, 11 Apr 2023 10:30:32 +0200 Subject: [PATCH 20/25] Remove unnecessary null check --- extensions/ql-vscode/src/view/results/result-tables.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ql-vscode/src/view/results/result-tables.tsx b/extensions/ql-vscode/src/view/results/result-tables.tsx index 9ae2e5613..ed3d5bae5 100644 --- a/extensions/ql-vscode/src/view/results/result-tables.tsx +++ b/extensions/ql-vscode/src/view/results/result-tables.tsx @@ -79,7 +79,7 @@ function renderResultCountString(resultSet: ResultSet): JSX.Element { } function getInterpretedTableName(interpretation: Interpretation): string { - return interpretation?.data.t === "GraphInterpretationData" + return interpretation.data.t === "GraphInterpretationData" ? GRAPH_TABLE_NAME : ALERTS_TABLE_NAME; } From 24c40af78f8a0c9e7651c84b59979488077f085b Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Tue, 11 Apr 2023 10:31:37 +0200 Subject: [PATCH 21/25] Add comment to @ts-ignore --- extensions/ql-vscode/src/view/results/result-tables.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ql-vscode/src/view/results/result-tables.tsx b/extensions/ql-vscode/src/view/results/result-tables.tsx index ed3d5bae5..24e60f1a0 100644 --- a/extensions/ql-vscode/src/view/results/result-tables.tsx +++ b/extensions/ql-vscode/src/view/results/result-tables.tsx @@ -101,7 +101,7 @@ function getResultSets( ): ResultSet[] { const resultSets: ResultSet[] = // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore 2783 + // @ts-ignore 2783 Avoid compilation error for overwriting the t property rawResultSets.map((rs) => ({ t: "RawResultSet", ...rs })); if (interpretation !== undefined) { From 9f3baadee989a005517b49350160368dc019683f Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Tue, 11 Apr 2023 11:10:45 +0200 Subject: [PATCH 22/25] Rename getResults to readQueryResults --- .../data-extensions-editor/data-extensions-editor-view.ts | 4 ++-- .../data-extensions-editor/external-api-usage-query.ts | 2 +- .../external-api-usage-query.test.ts | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 96857d4f9..7d8877af7 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -27,7 +27,7 @@ import { ResolvableLocationValue } from "../pure/bqrs-cli-types"; import { showResolvableLocation } from "../interface-utils"; import { decodeBqrsToExternalApiUsages } from "./bqrs"; import { redactableError } from "../pure/errors"; -import { getResults, runQuery } from "./external-api-usage-query"; +import { readQueryResults, runQuery } from "./external-api-usage-query"; import { createDataExtensionYaml, loadDataExtensionYaml } from "./yaml"; import { ExternalApiUsage } from "./external-api-usage"; import { ModeledMethod } from "./modeled-method"; @@ -192,7 +192,7 @@ export class DataExtensionsEditorView extends AbstractWebview< maxStep: 1500, }); - const bqrsChunk = await getResults({ + const bqrsChunk = await readQueryResults({ cliServer: this.cliServer, bqrsPath: queryResult.outputDir.bqrsPath, logger: extLogger, diff --git a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts index 9abe14c14..b2aa8633a 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts @@ -90,7 +90,7 @@ export type GetResultsOptions = { logger: Logger; }; -export async function getResults({ +export async function readQueryResults({ cliServer, bqrsPath, logger, diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts index 9d76be823..7351fb3fa 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts @@ -1,5 +1,5 @@ import { - getResults, + readQueryResults, runQuery, } from "../../../../src/data-extensions-editor/external-api-usage-query"; import { createMockLogger } from "../../../__mocks__/loggerMock"; @@ -135,7 +135,7 @@ describe("getResults", () => { "result-sets": [], }); - expect(await getResults(options)).toBeUndefined(); + expect(await readQueryResults(options)).toBeUndefined(); expect(options.logger.log).toHaveBeenCalledWith( expect.stringMatching(/Expected exactly one result set/), ); @@ -165,7 +165,7 @@ describe("getResults", () => { ], }); - expect(await getResults(options)).toBeUndefined(); + expect(await readQueryResults(options)).toBeUndefined(); expect(options.logger.log).toHaveBeenCalledWith( expect.stringMatching(/Expected exactly one result set/), ); @@ -211,7 +211,7 @@ describe("getResults", () => { }; options.cliServer.bqrsDecode.mockResolvedValue(decodedResultSet); - const result = await getResults(options); + const result = await readQueryResults(options); expect(result).toEqual(decodedResultSet); expect(options.cliServer.bqrsInfo).toHaveBeenCalledWith(options.bqrsPath); expect(options.cliServer.bqrsDecode).toHaveBeenCalledWith( From deb2b83642489ebc323092a69a73a96dba087b71 Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Tue, 11 Apr 2023 11:02:03 +0100 Subject: [PATCH 23/25] Update selected code flow when data flow paths change (#2288) --- extensions/ql-vscode/CHANGELOG.md | 1 + .../ql-vscode/src/view/data-flow-paths/DataFlowPaths.tsx | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md index 910758c11..ecd640ecd 100644 --- a/extensions/ql-vscode/CHANGELOG.md +++ b/extensions/ql-vscode/CHANGELOG.md @@ -2,6 +2,7 @@ ## [UNRELEASED] +- Fix bug that was causing code flows to not get updated when switching between results. [#2288](https://github.com/github/vscode-codeql/pull/2288) - Restart the CodeQL language server whenever the _CodeQL: Restart Query Server_ command is invoked. This avoids bugs where the CLI version changes to support new language features, but the language server is not updated. [#2238](https://github.com/github/vscode-codeql/pull/2238) ## 1.8.1 - 23 March 2023 diff --git a/extensions/ql-vscode/src/view/data-flow-paths/DataFlowPaths.tsx b/extensions/ql-vscode/src/view/data-flow-paths/DataFlowPaths.tsx index 7bccc0bed..e1e879e19 100644 --- a/extensions/ql-vscode/src/view/data-flow-paths/DataFlowPaths.tsx +++ b/extensions/ql-vscode/src/view/data-flow-paths/DataFlowPaths.tsx @@ -41,6 +41,11 @@ export const DataFlowPaths = ({ const { codeFlows, ruleDescription, message, severity } = dataFlowPaths; + React.useEffect(() => { + // Make sure to update the selected code flow if the data flow paths change + setSelectedCodeFlow(dataFlowPaths.codeFlows[0]); + }, [dataFlowPaths]); + return ( <> From ae08a1b598257c6a388d3af7d5a13b622961a872 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Tue, 11 Apr 2023 13:31:51 +0200 Subject: [PATCH 24/25] Use data extensions for finding external API calls The data extension editor was only using the default data extensions found in the `ql` submodule to find external API calls. This will add support for using data extensions found in the workspace. Rather than using the `codeQL.runningQueries.useExtensionPacks` setting, this will always include data extensions since the editor doesn't make sense to use without data extensions. We will also forbid the user from opening this view unless they are using a CLI which supports data extension packs. --- .../data-extensions-editor-module.ts | 13 ++++++++++--- .../external-api-usage-query.ts | 7 ++++++- .../external-api-usage-query.test.ts | 14 +++++++------- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts index c23a1462b..b2cce16ad 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts @@ -1,13 +1,13 @@ import { ExtensionContext } from "vscode"; import { DataExtensionsEditorView } from "./data-extensions-editor-view"; import { DataExtensionsEditorCommands } from "../common/commands"; -import { CodeQLCliServer } from "../cli"; +import { CliVersionConstraint, CodeQLCliServer } from "../cli"; import { QueryRunner } from "../queryRunner"; import { DatabaseManager } from "../local-databases"; -import { extLogger } from "../common"; import { ensureDir } from "fs-extra"; import { join } from "path"; import { App } from "../common/app"; +import { showAndLogErrorMessage } from "../helpers"; export class DataExtensionsEditorModule { private readonly queryStorageDir: string; @@ -52,7 +52,14 @@ export class DataExtensionsEditorModule { "codeQL.openDataExtensionsEditor": async () => { const db = this.databaseManager.currentDatabaseItem; if (!db) { - void extLogger.log("No database selected"); + void showAndLogErrorMessage("No database selected"); + return; + } + + if (!(await this.cliServer.cliConstraints.supportsQlpacksKind())) { + void showAndLogErrorMessage( + `This feature requires CodeQL CLI version ${CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND.format()} or later.`, + ); return; } diff --git a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts index b2aa8633a..a5cd3c052 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts @@ -54,6 +54,11 @@ export async function runQuery({ } await writeFile(suiteFile, dumpYaml(suiteYaml), "utf8"); + const additionalPacks = getOnDiskWorkspaceFolders(); + const extensionPacks = Object.keys( + await cliServer.resolveQlpacks(additionalPacks, true), + ); + const queries = await cliServer.resolveQueriesInSuite( suiteFile, getOnDiskWorkspaceFolders(), @@ -71,7 +76,7 @@ export async function runQuery({ { queryPath: query, quickEvalPosition: undefined }, false, getOnDiskWorkspaceFolders(), - undefined, + extensionPacks, queryStorageDir, undefined, undefined, diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts index 7351fb3fa..43a68cb15 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts @@ -36,11 +36,9 @@ describe("runQuery", () => { const options = { cliServer: { - resolveQlpacks: jest - .fn() - .mockRejectedValue( - new Error("Did not expect mocked method to be called"), - ), + resolveQlpacks: jest.fn().mockResolvedValue({ + "my/java-extensions": "/a/b/c/", + }), resolveQueriesInSuite: jest .fn() .mockResolvedValue([ @@ -103,6 +101,8 @@ describe("runQuery", () => { }, ]); + expect(options.cliServer.resolveQlpacks).toHaveBeenCalledTimes(1); + expect(options.cliServer.resolveQlpacks).toHaveBeenCalledWith([], true); expect(options.queryRunner.createQueryRun).toHaveBeenCalledWith( "/a/b/c/src.zip", { @@ -112,7 +112,7 @@ describe("runQuery", () => { }, false, [], - undefined, + ["my/java-extensions"], "/tmp/queries", undefined, undefined, @@ -120,7 +120,7 @@ describe("runQuery", () => { }); }); -describe("getResults", () => { +describe("readQueryResults", () => { const options = { cliServer: { bqrsInfo: jest.fn(), From 361cb60044d93af6ed185156dcbab8546e7db830 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 11 Apr 2023 12:45:48 +0100 Subject: [PATCH 25/25] Don't rate limit if no CLI is installed --- extensions/ql-vscode/src/distribution.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extensions/ql-vscode/src/distribution.ts b/extensions/ql-vscode/src/distribution.ts index 017da698d..0c8d9364f 100644 --- a/extensions/ql-vscode/src/distribution.ts +++ b/extensions/ql-vscode/src/distribution.ts @@ -215,6 +215,9 @@ export class DistributionManager implements DistributionProvider { minSecondsSinceLastUpdateCheck: number, ): Promise { const distribution = await this.getDistributionWithoutVersionCheck(); + if (distribution === undefined) { + minSecondsSinceLastUpdateCheck = 0; + } const extensionManagedCodeQlPath = await this.extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck(); if (distribution?.codeQlPath !== extensionManagedCodeQlPath) {