From 7f65122adb82fd078d3bce3e280ede682f39988d Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Tue, 4 Apr 2023 13:54:35 +0200 Subject: [PATCH 01/25] Add generating of flow model to data extension editor This adds the automatic generation of sources/sinks/summary flows to the data extension editor using the flow model queries. This is based on the Python script available in the CodeQL repo. See: https://github.com/github/codeql/blob/main/java/ql/src/utils/modelgenerator/GenerateFlowModel.py --- .../data-extensions-editor-module.ts | 4 + .../data-extensions-editor-view.ts | 92 ++++++++++- .../generate-flow-model.ts | 146 ++++++++++++++++++ .../src/data-extensions-editor/yaml.ts | 2 +- extensions/ql-vscode/src/extension.ts | 1 + .../ql-vscode/src/pure/interface-types.ts | 16 +- .../DataExtensionsEditor.tsx | 26 ++++ 7 files changed, 282 insertions(+), 5 deletions(-) create mode 100644 extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts index 2cbf6a53d..94a1620e8 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts @@ -5,10 +5,12 @@ import { CodeQLCliServer } from "../cli"; import { QueryRunner } from "../queryRunner"; import { DatabaseManager } from "../local-databases"; import { extLogger } from "../common"; +import { App } from "../common/app"; export class DataExtensionsEditorModule { public constructor( private readonly ctx: ExtensionContext, + private readonly app: App, private readonly databaseManager: DatabaseManager, private readonly cliServer: CodeQLCliServer, private readonly queryRunner: QueryRunner, @@ -26,6 +28,8 @@ export class DataExtensionsEditorModule { const view = new DataExtensionsEditorView( this.ctx, + this.app, + this.databaseManager, this.cliServer, this.queryRunner, this.queryStorageDir, diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 066b7aaec..2458bcb47 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -18,9 +18,13 @@ import { file } from "tmp-promise"; import { readFile, writeFile } from "fs-extra"; import { dump, load } from "js-yaml"; import { getOnDiskWorkspaceFolders } from "../helpers"; -import { DatabaseItem } from "../local-databases"; +import { DatabaseItem, DatabaseManager } from "../local-databases"; import { CodeQLCliServer } from "../cli"; -import { assertNever } from "../pure/helpers-pure"; +import { assertNever, getErrorMessage } from "../pure/helpers-pure"; +import { generateFlowModel } from "./generate-flow-model"; +import { ModeledMethod } from "./interface"; +import { promptImportGithubDatabase } from "../databaseFetcher"; +import { App } from "../common/app"; export class DataExtensionsEditorView extends AbstractWebview< ToDataExtensionsEditorMessage, @@ -28,6 +32,8 @@ export class DataExtensionsEditorView extends AbstractWebview< > { public constructor( ctx: ExtensionContext, + private readonly app: App, + private readonly databaseManager: DatabaseManager, private readonly cliServer: CodeQLCliServer, private readonly queryRunner: QueryRunner, private readonly queryStorageDir: string, @@ -69,6 +75,10 @@ export class DataExtensionsEditorView extends AbstractWebview< await this.saveYaml(msg.yaml); await this.loadExternalApiUsages(); + break; + case "generateExternalApi": + await this.generateExternalApi(); + break; default: assertNever(msg); @@ -149,6 +159,84 @@ export class DataExtensionsEditorView extends AbstractWebview< await this.clearProgress(); } + protected async generateExternalApi(): Promise { + const tokenSource = new CancellationTokenSource(); + + const selectedDatabase = this.databaseManager.currentDatabaseItem; + + const database = await promptImportGithubDatabase( + this.app.commands, + this.databaseManager, + this.app.workspaceStoragePath ?? this.app.globalStoragePath, + this.app.credentials, + (update) => this.showProgress(update), + tokenSource.token, + this.cliServer, + ); + if (!database) { + await this.clearProgress(); + void extLogger.log("No database chosen"); + + return; + } + + await this.databaseManager.setCurrentDatabaseItem(selectedDatabase); + + const workspaceFolder = workspace.workspaceFolders?.find( + (folder) => folder.name === "ql", + ); + if (!workspaceFolder) { + void extLogger.log("No workspace folder 'ql' found"); + + return; + } + + await this.showProgress({ + step: 0, + maxStep: 4000, + message: "Generating external API", + }); + + try { + await generateFlowModel( + this.cliServer, + this.queryRunner, + this.queryStorageDir, + workspaceFolder.uri.fsPath, + database, + async (results) => { + const modeledMethodsByName: Record = {}; + + for (const result of results) { + modeledMethodsByName[result[0]] = result[1]; + } + + await this.postMessage({ + t: "addModeledMethods", + modeledMethods: modeledMethodsByName, + }); + }, + (update) => this.showProgress(update), + tokenSource.token, + ); + } catch (e: unknown) { + void extLogger.log(`Error: ${getErrorMessage(e)}`); + } + + await this.databaseManager.removeDatabaseItem( + () => + this.showProgress({ + step: 3900, + maxStep: 4000, + message: "Removing temporary database", + }), + tokenSource.token, + database, + ); + + await this.clearProgress(); + } + private async runQuery(): Promise { const qlpacks = await qlpackOfDatabase(this.cliServer, this.databaseItem); diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts new file mode 100644 index 000000000..d60951817 --- /dev/null +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -0,0 +1,146 @@ +import { CancellationToken } from "vscode"; +import { DatabaseItem } from "../local-databases"; +import { ModeledMethod, ModeledMethodType } from "./interface"; +import { join } from "path"; +import { QueryRunner } from "../queryRunner"; +import { CodeQLCliServer } from "../cli"; +import { extLogger, TeeLogger } from "../common"; +import { definitions } from "./yaml"; +import { ProgressCallback } from "../progress"; +import { getOnDiskWorkspaceFolders } from "../helpers"; + +class FlowModelGenerator { + constructor( + private readonly cli: CodeQLCliServer, + private readonly queryRunner: QueryRunner, + private readonly queryStorageDir: string, + private readonly qlDir: string, + private readonly databaseItem: DatabaseItem, + private readonly progress: ProgressCallback, + private readonly token: CancellationToken, + ) {} + + async getAddsTo( + type: Exclude, + queryName: string, + queryStep: number, + ): Promise | undefined> { + const definition = definitions[type]; + + const query = join( + this.qlDir, + this.databaseItem.language, + "ql/src/utils/modelgenerator", + queryName, + ); + + const queryRun = this.queryRunner.createQueryRun( + this.databaseItem.databaseUri.fsPath, + { queryPath: query, quickEvalPosition: undefined }, + false, + getOnDiskWorkspaceFolders(), + undefined, + this.queryStorageDir, + undefined, + undefined, + ); + + const queryResult = await queryRun.evaluate( + ({ step, message }) => + this.progress({ + message: `Generating ${type} model: ${message}`, + step: queryStep * 1000 + step, + maxStep: 4000, + }), + this.token, + new TeeLogger(this.queryRunner.logger, queryRun.outputDir.logPath), + ); + + const bqrsPath = queryResult.outputDir.bqrsPath; + + const bqrsInfo = await this.cli.bqrsInfo(bqrsPath); + if (bqrsInfo["result-sets"].length !== 1) { + void extLogger.log( + `Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`, + ); + return undefined; + } + + const resultSet = bqrsInfo["result-sets"][0]; + + const decodedResults = await this.cli.bqrsDecode(bqrsPath, resultSet.name); + + const results = decodedResults.tuples; + + return results + .map((result): [string, ModeledMethod] | undefined => { + const row = result[0] as string; + + return definition.readModeledMethod(row.split(";")); + }) + .filter( + (result): result is [string, ModeledMethod] => result !== undefined, + ); + } + + async run( + onResults: ( + results: Array<[string, ModeledMethod]>, + ) => void | Promise, + ) { + const summaryResults = await this.getAddsTo( + "summary", + "CaptureSummaryModels.ql", + 0, + ); + if (summaryResults) { + await onResults(summaryResults); + } + + const sinkResults = await this.getAddsTo("sink", "CaptureSinkModels.ql", 1); + if (sinkResults) { + await onResults(sinkResults); + } + + const sourceResults = await this.getAddsTo( + "source", + "CaptureSourceModels.ql", + 2, + ); + if (sourceResults) { + await onResults(sourceResults); + } + + const neutralResults = await this.getAddsTo( + "neutral", + "CaptureNeutralModels.ql", + 3, + ); + if (neutralResults) { + await onResults(neutralResults); + } + } +} + +export async function generateFlowModel( + cli: CodeQLCliServer, + queryRunner: QueryRunner, + queryStorageDir: string, + qlDir: string, + databaseItem: DatabaseItem, + onResults: (results: Array<[string, ModeledMethod]>) => void | Promise, + progress: ProgressCallback, + token: CancellationToken, +) { + const generator = new FlowModelGenerator( + cli, + queryRunner, + queryStorageDir, + qlDir, + databaseItem, + progress, + token, + ); + + return generator.run(onResults); +} diff --git a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts index b8ff007c4..31bf3ef45 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts @@ -19,7 +19,7 @@ function readRowToMethod(row: any[]): string { return `${row[0]}.${row[1]}#${row[3]}${row[4]}`; } -const definitions: Record< +export const definitions: Record< Exclude, DataExtensionDefinition > = { diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index db25f3195..1adb70751 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -868,6 +868,7 @@ async function activateWithInstalledDistribution( await ensureDir(dataExtensionsEditorQueryStorageDir); const dataExtensionsEditorModule = new DataExtensionsEditorModule( ctx, + app, dbm, cliServer, qs, diff --git a/extensions/ql-vscode/src/pure/interface-types.ts b/extensions/ql-vscode/src/pure/interface-types.ts index b295695bd..de4828af2 100644 --- a/extensions/ql-vscode/src/pure/interface-types.ts +++ b/extensions/ql-vscode/src/pure/interface-types.ts @@ -15,6 +15,7 @@ import { import { RepositoriesFilterSortStateWithIds } from "./variant-analysis-filter-sort"; import { ErrorLike } from "./errors"; import { DataFlowPaths } from "../variant-analysis/shared/data-flow-paths"; +import { ModeledMethod } from "../data-extensions-editor/interface"; /** * This module contains types and code that are shared between @@ -497,16 +498,27 @@ export interface SetExistingYamlDataMessage { data: any; } +export interface AddModeledMethodsMessage { + t: "addModeledMethods"; + modeledMethods: Record; +} + export interface ApplyDataExtensionYamlMessage { t: "applyDataExtensionYaml"; yaml: string; } +export interface GenerateExternalApiMessage { + t: "generateExternalApi"; +} + export type ToDataExtensionsEditorMessage = | SetExternalApiResultsMessage | ShowProgressMessage - | SetExistingYamlDataMessage; + | SetExistingYamlDataMessage + | AddModeledMethodsMessage; export type FromDataExtensionsEditorMessage = | ViewLoadedMsg - | ApplyDataExtensionYamlMessage; + | ApplyDataExtensionYamlMessage + | GenerateExternalApiMessage; diff --git a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx index 86ae620fb..a241660d7 100644 --- a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx +++ b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx @@ -74,6 +74,20 @@ export function DataExtensionsEditor(): JSX.Element { }; }); + break; + case "addModeledMethods": + setModeledMethods((oldModeledMethods) => { + const filteredOldModeledMethods = Object.fromEntries( + Object.entries(oldModeledMethods).filter( + ([, value]) => value.type !== "none", + ), + ); + + return { + ...msg.modeledMethods, + ...filteredOldModeledMethods, + }; + }); break; default: assertNever(msg); @@ -168,6 +182,12 @@ export function DataExtensionsEditor(): JSX.Element { }); }, [methods, modeledMethods]); + const onGenerateClick = useCallback(() => { + vscode.postMessage({ + t: "generateExternalApi", + }); + }, []); + return ( {progress.maxStep > 0 && ( @@ -189,6 +209,12 @@ export function DataExtensionsEditor(): JSX.Element {

External API modelling

Apply +   + + Download and generate + +
+
From a1809b0bac6b1ab17f494f3c39220af26c077f1e Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 11:22:52 +0200 Subject: [PATCH 02/25] Merge similar messages for adding modeled methods --- .../data-extensions-editor-view.ts | 5 +++-- .../ql-vscode/src/pure/interface-types.ts | 14 +++++++------ .../DataExtensionsEditor.tsx | 21 +++++++------------ 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 1c4e04082..b3a022370 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -142,8 +142,8 @@ export class DataExtensionsEditorView extends AbstractWebview< } await this.postMessage({ - t: "setExistingModeledMethods", - existingModeledMethods, + t: "addModeledMethods", + modeledMethods: existingModeledMethods, }); } catch (e: unknown) { void extLogger.log(`Unable to read data extension YAML: ${e}`); @@ -250,6 +250,7 @@ export class DataExtensionsEditorView extends AbstractWebview< await this.postMessage({ t: "addModeledMethods", modeledMethods: modeledMethodsByName, + overrideNone: true, }); }, (update) => this.showProgress(update), diff --git a/extensions/ql-vscode/src/pure/interface-types.ts b/extensions/ql-vscode/src/pure/interface-types.ts index bc5599cf0..06ae988c6 100644 --- a/extensions/ql-vscode/src/pure/interface-types.ts +++ b/extensions/ql-vscode/src/pure/interface-types.ts @@ -493,14 +493,17 @@ export interface ShowProgressMessage { message: string; } -export interface SetExistingModeledMethods { - t: "setExistingModeledMethods"; - existingModeledMethods: Record; -} - export interface AddModeledMethodsMessage { t: "addModeledMethods"; modeledMethods: Record; + + /** + * If true, then any existing modeled methods set to "none" will be + * overwritten by the new modeled methods. Otherwise, the "none" modeled + * methods will not be overwritten, even if the new modeled methods + * contain a better model. + */ + overrideNone?: boolean; } export interface SaveModeledMethods { @@ -516,7 +519,6 @@ export interface GenerateExternalApiMessage { export type ToDataExtensionsEditorMessage = | SetExternalApiUsagesMessage | ShowProgressMessage - | SetExistingModeledMethods | AddModeledMethodsMessage; export type FromDataExtensionsEditorMessage = diff --git a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx index d757c6b89..594d4f898 100644 --- a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx +++ b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx @@ -56,23 +56,16 @@ export function DataExtensionsEditor(): JSX.Element { break; case "showProgress": setProgress(msg); - break; - case "setExistingModeledMethods": - setModeledMethods((oldModeledMethods) => { - return { - ...msg.existingModeledMethods, - ...oldModeledMethods, - }; - }); - break; case "addModeledMethods": setModeledMethods((oldModeledMethods) => { - const filteredOldModeledMethods = Object.fromEntries( - Object.entries(oldModeledMethods).filter( - ([, value]) => value.type !== "none", - ), - ); + const filteredOldModeledMethods = msg.overrideNone + ? Object.fromEntries( + Object.entries(oldModeledMethods).filter( + ([, value]) => value.type !== "none", + ), + ) + : oldModeledMethods; return { ...msg.modeledMethods, From 14ad348f24f6ff1c70928f723a6c13c74140c6d0 Mon Sep 17 00:00:00 2001 From: Nora Date: Wed, 5 Apr 2023 10:09:40 +0000 Subject: [PATCH 03/25] new repo states data type and mapping --- .../store/repo-states-data-types.ts | 7 +++++ .../store/repo-states-store.ts | 28 +++++++++++++++++-- .../variant-analysis-manager.ts | 11 +++++--- 3 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 extensions/ql-vscode/src/variant-analysis/store/repo-states-data-types.ts diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-states-data-types.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-states-data-types.ts new file mode 100644 index 000000000..8fff69bc3 --- /dev/null +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-states-data-types.ts @@ -0,0 +1,7 @@ +import { VariantAnalysisScannedRepositoryDownloadStatus } from "../shared/variant-analysis"; + +export interface VariantAnalysisScannedRepositoryStateData { + repositoryId: number; + downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus; + downloadPercentage?: number; +} diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts index 3afc23811..53b72e921 100644 --- a/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts @@ -1,17 +1,39 @@ import { outputJson, readJson } from "fs-extra"; import { VariantAnalysisScannedRepositoryState } from "../shared/variant-analysis"; +import { VariantAnalysisScannedRepositoryStateData } from "./repo-states-data-types"; export const REPO_STATES_FILENAME = "repo_states.json"; export async function writeRepoStates( storagePath: string, - repoStates: Record | undefined, + repoStates: Record, ): Promise { - return await outputJson(storagePath, repoStates); + // Map from repoStates Domain type to the repoStates Data type + const repoStatesData = Object.fromEntries( + Object.entries(repoStates).map(([key, value]) => { + const dataItem: VariantAnalysisScannedRepositoryStateData = value; + return [key, dataItem]; + }), + ); + + return await outputJson(storagePath, repoStatesData); } export async function readRepoStates( storagePath: string, ): Promise> { - return await readJson(storagePath); + const repoStatesData: Record< + number, + VariantAnalysisScannedRepositoryStateData + > = await readJson(storagePath); + + // Map from repoStates Data type to the repoStates Domain type + const repoStates = Object.fromEntries( + Object.entries(repoStatesData).map(([key, value]) => { + const dataItem: VariantAnalysisScannedRepositoryState = value; + return [key, dataItem]; + }), + ); + + return repoStates; } diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts index c382b87d2..e265707b2 100644 --- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts +++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts @@ -598,10 +598,13 @@ export class VariantAnalysisManager VariantAnalysisScannedRepositoryDownloadStatus.Succeeded; await this.onRepoStateUpdated(variantAnalysis.id, repoState); - await writeRepoStates( - this.getRepoStatesStoragePath(variantAnalysis.id), - this.repoStates.get(variantAnalysis.id), - ); + const repoStates = this.repoStates.get(variantAnalysis.id); + if (repoStates) { + await writeRepoStates( + this.getRepoStatesStoragePath(variantAnalysis.id), + repoStates, + ); + } } public async enqueueDownload( From 3f5bc85004d8d4a7ab8741d37903fe7557af2da2 Mon Sep 17 00:00:00 2001 From: Nora Date: Thu, 6 Apr 2023 08:39:30 +0000 Subject: [PATCH 04/25] Map enum properly --- .../store/repo-states-data-types.ts | 11 ++++-- .../store/repo-states-store.ts | 8 ++--- .../store/repo-states-to-data-mapper.ts | 35 +++++++++++++++++++ .../store/repo-states-to-domain-mapper.ts | 35 +++++++++++++++++++ 4 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 extensions/ql-vscode/src/variant-analysis/store/repo-states-to-data-mapper.ts create mode 100644 extensions/ql-vscode/src/variant-analysis/store/repo-states-to-domain-mapper.ts diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-states-data-types.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-states-data-types.ts index 8fff69bc3..17d6fc334 100644 --- a/extensions/ql-vscode/src/variant-analysis/store/repo-states-data-types.ts +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-states-data-types.ts @@ -1,7 +1,12 @@ -import { VariantAnalysisScannedRepositoryDownloadStatus } from "../shared/variant-analysis"; - export interface VariantAnalysisScannedRepositoryStateData { repositoryId: number; - downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus; + downloadStatus: VariantAnalysisScannedRepositoryDownloadData; downloadPercentage?: number; } + +export enum VariantAnalysisScannedRepositoryDownloadData { + Pending = "pending", + InProgress = "inProgress", + Succeeded = "succeeded", + Failed = "failed", +} diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts index 53b72e921..5994df622 100644 --- a/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts @@ -1,6 +1,8 @@ import { outputJson, readJson } from "fs-extra"; import { VariantAnalysisScannedRepositoryState } from "../shared/variant-analysis"; import { VariantAnalysisScannedRepositoryStateData } from "./repo-states-data-types"; +import { mapRepoStateToData } from "./repo-states-to-data-mapper"; +import { mapRepoStateToDomain } from "./repo-states-to-domain-mapper"; export const REPO_STATES_FILENAME = "repo_states.json"; @@ -11,8 +13,7 @@ export async function writeRepoStates( // Map from repoStates Domain type to the repoStates Data type const repoStatesData = Object.fromEntries( Object.entries(repoStates).map(([key, value]) => { - const dataItem: VariantAnalysisScannedRepositoryStateData = value; - return [key, dataItem]; + return [key, mapRepoStateToData(value)]; }), ); @@ -30,8 +31,7 @@ export async function readRepoStates( // Map from repoStates Data type to the repoStates Domain type const repoStates = Object.fromEntries( Object.entries(repoStatesData).map(([key, value]) => { - const dataItem: VariantAnalysisScannedRepositoryState = value; - return [key, dataItem]; + return [key, mapRepoStateToDomain(value)]; }), ); diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-states-to-data-mapper.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-states-to-data-mapper.ts new file mode 100644 index 000000000..39fcfc989 --- /dev/null +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-states-to-data-mapper.ts @@ -0,0 +1,35 @@ +import { + VariantAnalysisScannedRepositoryDownloadStatus, + VariantAnalysisScannedRepositoryState, +} from "../shared/variant-analysis"; +import { + VariantAnalysisScannedRepositoryDownloadData, + VariantAnalysisScannedRepositoryStateData, +} from "./repo-states-data-types"; + +export function mapRepoStateToData( + repoState: VariantAnalysisScannedRepositoryState, +): VariantAnalysisScannedRepositoryStateData { + return { + repositoryId: repoState.repositoryId, + downloadStatus: processDownloadStatus(repoState.downloadStatus), + downloadPercentage: repoState.downloadPercentage, + }; +} + +function processDownloadStatus( + downloadedStatus: VariantAnalysisScannedRepositoryDownloadStatus, +) { + switch (downloadedStatus) { + case VariantAnalysisScannedRepositoryDownloadStatus.Pending: + return VariantAnalysisScannedRepositoryDownloadData.Pending; + case VariantAnalysisScannedRepositoryDownloadStatus.InProgress: + return VariantAnalysisScannedRepositoryDownloadData.InProgress; + case VariantAnalysisScannedRepositoryDownloadStatus.Succeeded: + return VariantAnalysisScannedRepositoryDownloadData.Succeeded; + case VariantAnalysisScannedRepositoryDownloadStatus.Failed: + return VariantAnalysisScannedRepositoryDownloadData.Failed; + default: + return VariantAnalysisScannedRepositoryDownloadData.Pending; + } +} diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-states-to-domain-mapper.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-states-to-domain-mapper.ts new file mode 100644 index 000000000..8dcdd2235 --- /dev/null +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-states-to-domain-mapper.ts @@ -0,0 +1,35 @@ +import { + VariantAnalysisScannedRepositoryState, + VariantAnalysisScannedRepositoryDownloadStatus, +} from "../shared/variant-analysis"; +import { + VariantAnalysisScannedRepositoryStateData, + VariantAnalysisScannedRepositoryDownloadData, +} from "./repo-states-data-types"; + +export function mapRepoStateToDomain( + repoState: VariantAnalysisScannedRepositoryStateData, +): VariantAnalysisScannedRepositoryState { + return { + repositoryId: repoState.repositoryId, + downloadStatus: processDownloadStatus(repoState.downloadStatus), + downloadPercentage: repoState.downloadPercentage, + }; +} + +function processDownloadStatus( + downloadedStatus: VariantAnalysisScannedRepositoryDownloadData, +) { + switch (downloadedStatus) { + case VariantAnalysisScannedRepositoryDownloadData.Pending: + return VariantAnalysisScannedRepositoryDownloadStatus.Pending; + case VariantAnalysisScannedRepositoryDownloadData.InProgress: + return VariantAnalysisScannedRepositoryDownloadStatus.InProgress; + case VariantAnalysisScannedRepositoryDownloadData.Succeeded: + return VariantAnalysisScannedRepositoryDownloadStatus.Succeeded; + case VariantAnalysisScannedRepositoryDownloadData.Failed: + return VariantAnalysisScannedRepositoryDownloadStatus.Failed; + default: + return VariantAnalysisScannedRepositoryDownloadStatus.Pending; + } +} From 8eaf1e9adcbac58874f9da0077e94b78e11744f8 Mon Sep 17 00:00:00 2001 From: Nora Date: Thu, 6 Apr 2023 10:07:41 +0000 Subject: [PATCH 05/25] Retain error message --- .../store/repo-states-store.ts | 29 +++++++++++-------- .../variant-analysis-manager.ts | 14 ++++----- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts index 5994df622..ce517f4ce 100644 --- a/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts @@ -22,18 +22,23 @@ export async function writeRepoStates( export async function readRepoStates( storagePath: string, -): Promise> { - const repoStatesData: Record< - number, - VariantAnalysisScannedRepositoryStateData - > = await readJson(storagePath); +): Promise | undefined> { + try { + const repoStatesData: Record< + number, + VariantAnalysisScannedRepositoryStateData + > = await readJson(storagePath); - // Map from repoStates Data type to the repoStates Domain type - const repoStates = Object.fromEntries( - Object.entries(repoStatesData).map(([key, value]) => { - return [key, mapRepoStateToDomain(value)]; - }), - ); + // Map from repoStates Data type to the repoStates Domain type + const repoStates = Object.fromEntries( + Object.entries(repoStatesData).map(([key, value]) => { + return [key, mapRepoStateToDomain(value)]; + }), + ); - return repoStates; + return repoStates; + } catch (e) { + // Ignore this error, we simply might not have downloaded anything yet + return undefined; + } } diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts index e265707b2..a1f1a9e71 100644 --- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts +++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts @@ -264,13 +264,13 @@ export class VariantAnalysisManager } else { await this.setVariantAnalysis(variantAnalysis); - try { - const repoStates = await readRepoStates( - this.getRepoStatesStoragePath(variantAnalysis.id), - ); - this.repoStates.set(variantAnalysis.id, repoStates); - } catch (e) { - // Ignore this error, we simply might not have downloaded anything yet + const repoStatesFromDisk = await readRepoStates( + this.getRepoStatesStoragePath(variantAnalysis.id), + ); + + if (repoStatesFromDisk) { + this.repoStates.set(variantAnalysis.id, repoStatesFromDisk); + } else { this.repoStates.set(variantAnalysis.id, {}); } From 7e3c0265fe5f5d4a7137ff8755ab5924690feec3 Mon Sep 17 00:00:00 2001 From: Nora Date: Thu, 6 Apr 2023 12:49:14 +0000 Subject: [PATCH 06/25] Use new methods in test --- .../variant-analysis-manager.test.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-manager.test.ts b/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-manager.test.ts index 9bc16707b..e68078722 100644 --- a/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-manager.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-manager.test.ts @@ -45,6 +45,10 @@ import { App } from "../../../../src/common/app"; import { ExtensionApp } from "../../../../src/common/vscode/vscode-app"; import { DbConfigStore } from "../../../../src/databases/config/db-config-store"; import { mockedObject } from "../../utils/mocking.helpers"; +import { + REPO_STATES_FILENAME, + writeRepoStates, +} from "../../../../src/variant-analysis/store/repo-states-store"; // up to 3 minutes per test jest.setTimeout(3 * 60 * 1000); @@ -119,8 +123,12 @@ describe("Variant Analysis Manager", () => { }); it("should read in the repo states if it exists", async () => { - await fs.writeJson( - join(storagePath, variantAnalysis.id.toString(), "repo_states.json"), + await writeRepoStates( + join( + storagePath, + variantAnalysis.id.toString(), + REPO_STATES_FILENAME, + ), { [scannedRepos[0].repository.id]: { repositoryId: scannedRepos[0].repository.id, @@ -177,7 +185,7 @@ describe("Variant Analysis Manager", () => { repoStatesPath = join( storagePath, variantAnalysis.id.toString(), - "repo_states.json", + REPO_STATES_FILENAME, ); }); From 103070ee7ac817ebd405fc24ba5da0767a0128dd Mon Sep 17 00:00:00 2001 From: Nora Date: Thu, 6 Apr 2023 12:52:45 +0000 Subject: [PATCH 07/25] Use shorthand --- .../src/variant-analysis/variant-analysis-manager.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts index a1f1a9e71..2dfc49d78 100644 --- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts +++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts @@ -268,11 +268,7 @@ export class VariantAnalysisManager this.getRepoStatesStoragePath(variantAnalysis.id), ); - if (repoStatesFromDisk) { - this.repoStates.set(variantAnalysis.id, repoStatesFromDisk); - } else { - this.repoStates.set(variantAnalysis.id, {}); - } + this.repoStates.set(variantAnalysis.id, repoStatesFromDisk || {}); if ( !(await isVariantAnalysisComplete( From 6c95ac7c790d033bc286e9201667329c6aa6c7c9 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 14:57:58 +0200 Subject: [PATCH 08/25] Fix error when closing MRVA webview during extension activation This fixes the "Webview is disposed" error which occurs when the user closes the variant analysis webview while the extension is still activating. We will now check whether the webview is disposed before restoring the view. --- .../variant-analysis-view-serializer.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-view-serializer.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-view-serializer.ts index 314a3e0ba..7ce93f372 100644 --- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-view-serializer.ts +++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-view-serializer.ts @@ -37,6 +37,14 @@ export class VariantAnalysisViewSerializer implements WebviewPanelSerializer { return; } + // Between the time the webview is deserialized and the time the extension + // is fully activated, the user may close the webview. In this case, we + // should not attempt to restore the view. + let disposed = false; + const unregisterOnDidDispose = webviewPanel.onDidDispose(() => { + disposed = true; + }); + const variantAnalysisState: VariantAnalysisState = state as VariantAnalysisState; @@ -46,11 +54,16 @@ export class VariantAnalysisViewSerializer implements WebviewPanelSerializer { variantAnalysisState.variantAnalysisId, ); if (existingView) { + unregisterOnDidDispose.dispose(); await existingView.openView(); webviewPanel.dispose(); return; } + if (disposed) { + return; + } + const view = new VariantAnalysisView( this.ctx, this.app, @@ -58,6 +71,8 @@ export class VariantAnalysisViewSerializer implements WebviewPanelSerializer { manager, ); await view.restoreView(webviewPanel); + + unregisterOnDidDispose.dispose(); } private waitForExtensionFullyLoaded(): Promise< From 1595d6f4aa3784eb7f84504f0511a83c66f1cde4 Mon Sep 17 00:00:00 2001 From: Nora Date: Thu, 6 Apr 2023 12:59:30 +0000 Subject: [PATCH 09/25] Fix default and use assertNever --- .../src/variant-analysis/store/repo-states-to-data-mapper.ts | 3 ++- .../src/variant-analysis/store/repo-states-to-domain-mapper.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-states-to-data-mapper.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-states-to-data-mapper.ts index 39fcfc989..a6853eeb1 100644 --- a/extensions/ql-vscode/src/variant-analysis/store/repo-states-to-data-mapper.ts +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-states-to-data-mapper.ts @@ -1,3 +1,4 @@ +import { assertNever } from "../../pure/helpers-pure"; import { VariantAnalysisScannedRepositoryDownloadStatus, VariantAnalysisScannedRepositoryState, @@ -30,6 +31,6 @@ function processDownloadStatus( case VariantAnalysisScannedRepositoryDownloadStatus.Failed: return VariantAnalysisScannedRepositoryDownloadData.Failed; default: - return VariantAnalysisScannedRepositoryDownloadData.Pending; + assertNever(downloadedStatus); } } diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-states-to-domain-mapper.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-states-to-domain-mapper.ts index 8dcdd2235..928d2f9c5 100644 --- a/extensions/ql-vscode/src/variant-analysis/store/repo-states-to-domain-mapper.ts +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-states-to-domain-mapper.ts @@ -1,3 +1,4 @@ +import { assertNever } from "../../pure/helpers-pure"; import { VariantAnalysisScannedRepositoryState, VariantAnalysisScannedRepositoryDownloadStatus, @@ -30,6 +31,6 @@ function processDownloadStatus( case VariantAnalysisScannedRepositoryDownloadData.Failed: return VariantAnalysisScannedRepositoryDownloadStatus.Failed; default: - return VariantAnalysisScannedRepositoryDownloadStatus.Pending; + assertNever(downloadedStatus); } } From 4fa229f8a2295c1ab5052d2148d55a8c54aa6327 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:16:37 +0200 Subject: [PATCH 10/25] Rename generateExternalApi to generateModeledMethods --- .../src/data-extensions-editor/data-extensions-editor-view.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index cccae294a..e0d05db85 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -95,7 +95,7 @@ export class DataExtensionsEditorView extends AbstractWebview< break; case "generateExternalApi": - await this.generateExternalApi(); + await this.generateModeledMethods(); break; default: @@ -222,7 +222,7 @@ export class DataExtensionsEditorView extends AbstractWebview< } } - protected async generateExternalApi(): Promise { + protected async generateModeledMethods(): Promise { const tokenSource = new CancellationTokenSource(); const selectedDatabase = this.databaseManager.currentDatabaseItem; From 42d40347442205590f548fe9a242dc266f09ac4f Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:20:01 +0200 Subject: [PATCH 11/25] Extract getting ql submodule folder to function --- .../data-extensions-editor-view.ts | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index e0d05db85..ec922f238 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -5,6 +5,7 @@ import { ViewColumn, window, workspace, + WorkspaceFolder, } from "vscode"; import { AbstractWebview, WebviewPanelConfig } from "../abstract-webview"; import { @@ -37,6 +38,19 @@ import { createDataExtensionYaml, loadDataExtensionYaml } from "./yaml"; import { ExternalApiUsage } from "./external-api-usage"; import { ModeledMethod } from "./modeled-method"; +function getQlSubmoduleFolder(): WorkspaceFolder | undefined { + const workspaceFolder = workspace.workspaceFolders?.find( + (folder) => folder.name === "ql", + ); + if (!workspaceFolder) { + void extLogger.log("No workspace folder 'ql' found"); + + return; + } + + return workspaceFolder; +} + export class DataExtensionsEditorView extends AbstractWebview< ToDataExtensionsEditorMessage, FromDataExtensionsEditorMessage @@ -245,12 +259,8 @@ export class DataExtensionsEditorView extends AbstractWebview< await this.databaseManager.setCurrentDatabaseItem(selectedDatabase); - const workspaceFolder = workspace.workspaceFolders?.find( - (folder) => folder.name === "ql", - ); + const workspaceFolder = getQlSubmoduleFolder(); if (!workspaceFolder) { - void extLogger.log("No workspace folder 'ql' found"); - return; } @@ -404,12 +414,8 @@ export class DataExtensionsEditorView extends AbstractWebview< } private calculateModelFilename(): string | undefined { - const workspaceFolder = workspace.workspaceFolders?.find( - (folder) => folder.name === "ql", - ); + const workspaceFolder = getQlSubmoduleFolder(); if (!workspaceFolder) { - void extLogger.log("No workspace folder 'ql' found"); - return; } From d3ff87ab71116fa196055593cc62a1862c9802ba Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:21:48 +0200 Subject: [PATCH 12/25] Update language used for generating modeled methods --- .../src/data-extensions-editor/data-extensions-editor-view.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index ec922f238..9a41ce59c 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -267,7 +267,7 @@ export class DataExtensionsEditorView extends AbstractWebview< await this.showProgress({ step: 0, maxStep: 4000, - message: "Generating external API", + message: "Generating modeled methods for library", }); try { From 5a3a1a5cd7633796201831dff45767a0c4e496a2 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:23:10 +0200 Subject: [PATCH 13/25] Add comment for flow model progress --- .../data-extensions-editor/data-extensions-editor-view.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 9a41ce59c..a4383b23a 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -395,6 +395,13 @@ export class DataExtensionsEditorView extends AbstractWebview< * that there's 1000 steps of the query progress since that takes the most time, and then * an additional 500 steps for the rest of the work. The progress doesn't need to be 100% * accurate, so this is just a rough estimate. + * + * For generating the modeled methods for an external library, the max step is 4000. This is + * based on the following steps: + * - 1000 for the summary model + * - 1000 for the sink model + * - 1000 for the source model + * - 1000 for the neutral model */ private async showProgress(update: ProgressUpdate, maxStep?: number) { await this.postMessage({ From 3214a376ad046d158216c6b31632b7253c8509b7 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:24:25 +0200 Subject: [PATCH 14/25] Clarify definitions to be extensible predicate definitions --- .../generate-flow-model.ts | 4 +-- .../src/data-extensions-editor/yaml.ts | 26 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index d13bf00fc..6eb71ab18 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -4,7 +4,7 @@ import { join } from "path"; import { QueryRunner } from "../queryRunner"; import { CodeQLCliServer } from "../cli"; import { extLogger, TeeLogger } from "../common"; -import { definitions } from "./yaml"; +import { extensiblePredicateDefinitions } from "./yaml"; import { ProgressCallback } from "../progress"; import { getOnDiskWorkspaceFolders } from "../helpers"; import { ModeledMethod, ModeledMethodType } from "./modeled-method"; @@ -25,7 +25,7 @@ class FlowModelGenerator { queryName: string, queryStep: number, ): Promise | undefined> { - const definition = definitions[type]; + const definition = extensiblePredicateDefinitions[type]; const query = join( this.qlDir, diff --git a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts index 7d72e434d..de7b60350 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts @@ -6,8 +6,8 @@ type ExternalApiUsageByType = { modeledMethod: ModeledMethod; }; -type DataExtensionDefinition = { - extensible: string; +type ExtensiblePredicateDefinition = { + extensiblePredicate: string; generateMethodDefinition: (method: ExternalApiUsageByType) => any[]; readModeledMethod: (row: any[]) => [string, ModeledMethod] | undefined; }; @@ -16,12 +16,12 @@ function readRowToMethod(row: any[]): string { return `${row[0]}.${row[1]}#${row[3]}${row[4]}`; } -export const definitions: Record< +export const extensiblePredicateDefinitions: Record< Exclude, - DataExtensionDefinition + ExtensiblePredicateDefinition > = { source: { - extensible: "sourceModel", + extensiblePredicate: "sourceModel", // extensible predicate sourceModel( // string package, string type, boolean subtypes, string name, string signature, string ext, // string output, string kind, string provenance @@ -48,7 +48,7 @@ export const definitions: Record< ], }, sink: { - extensible: "sinkModel", + extensiblePredicate: "sinkModel", // extensible predicate sinkModel( // string package, string type, boolean subtypes, string name, string signature, string ext, // string input, string kind, string provenance @@ -75,7 +75,7 @@ export const definitions: Record< ], }, summary: { - extensible: "summaryModel", + extensiblePredicate: "summaryModel", // extensible predicate summaryModel( // string package, string type, boolean subtypes, string name, string signature, string ext, // string input, string output, string kind, string provenance @@ -103,7 +103,7 @@ export const definitions: Record< ], }, neutral: { - extensible: "neutralModel", + extensiblePredicate: "neutralModel", // extensible predicate neutralModel( // string package, string type, string name, string signature, string provenance // ); @@ -128,7 +128,7 @@ export const definitions: Record< function createDataProperty( methods: ExternalApiUsageByType[], - definition: DataExtensionDefinition, + definition: ExtensiblePredicateDefinition, ) { if (methods.length === 0) { return " []"; @@ -169,10 +169,10 @@ export function createDataExtensionYaml( } } - const extensions = Object.entries(definitions).map( + const extensions = Object.entries(extensiblePredicateDefinitions).map( ([type, definition]) => ` - addsTo: pack: codeql/java-all - extensible: ${definition.extensible} + extensible: ${definition.extensiblePredicate} data:${createDataProperty( methodsByType[type as Exclude], definition, @@ -214,8 +214,8 @@ export function loadDataExtensionYaml( continue; } - const definition = Object.values(definitions).find( - (definition) => definition.extensible === extensible, + const definition = Object.values(extensiblePredicateDefinitions).find( + (definition) => definition.extensiblePredicate === extensible, ); if (!definition) { continue; From 0d00e5c5b7054bb9dfc4f53ae456a18984b8311e Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:27:34 +0200 Subject: [PATCH 15/25] Remove undefined from readModeledMethod return type --- .../data-extensions-editor/generate-flow-model.ts | 12 ++++-------- .../ql-vscode/src/data-extensions-editor/yaml.ts | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index 6eb71ab18..1ab535b69 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -72,15 +72,11 @@ class FlowModelGenerator { const results = decodedResults.tuples; - return results - .map((result): [string, ModeledMethod] | undefined => { - const row = result[0] as string; + return results.map((result): [string, ModeledMethod] => { + const row = result[0] as string; - return definition.readModeledMethod(row.split(";")); - }) - .filter( - (result): result is [string, ModeledMethod] => result !== undefined, - ); + return definition.readModeledMethod(row.split(";")); + }); } async run( diff --git a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts index de7b60350..c573d1f88 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts @@ -9,7 +9,7 @@ type ExternalApiUsageByType = { type ExtensiblePredicateDefinition = { extensiblePredicate: string; generateMethodDefinition: (method: ExternalApiUsageByType) => any[]; - readModeledMethod: (row: any[]) => [string, ModeledMethod] | undefined; + readModeledMethod: (row: any[]) => [string, ModeledMethod]; }; function readRowToMethod(row: any[]): string { From d5403ad92656b7f838753c424127c1d93ab2d6ca Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:33:43 +0200 Subject: [PATCH 16/25] Introduce type for modeled method with signature --- .../data-extensions-editor-view.ts | 2 +- .../generate-flow-model.ts | 15 ++++--- .../data-extensions-editor/modeled-method.ts | 5 +++ .../src/data-extensions-editor/yaml.ts | 44 ++++++++++--------- 4 files changed, 38 insertions(+), 28 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index a4383b23a..1622f6875 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -281,7 +281,7 @@ export class DataExtensionsEditorView extends AbstractWebview< const modeledMethodsByName: Record = {}; for (const result of results) { - modeledMethodsByName[result[0]] = result[1]; + modeledMethodsByName[result.signature] = result.modeledMethod; } await this.postMessage({ diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index 1ab535b69..8f737fe3e 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -7,7 +7,10 @@ import { extLogger, TeeLogger } from "../common"; import { extensiblePredicateDefinitions } from "./yaml"; import { ProgressCallback } from "../progress"; import { getOnDiskWorkspaceFolders } from "../helpers"; -import { ModeledMethod, ModeledMethodType } from "./modeled-method"; +import { + ModeledMethodType, + ModeledMethodWithSignature, +} from "./modeled-method"; class FlowModelGenerator { constructor( @@ -24,7 +27,7 @@ class FlowModelGenerator { type: Exclude, queryName: string, queryStep: number, - ): Promise | undefined> { + ): Promise { const definition = extensiblePredicateDefinitions[type]; const query = join( @@ -72,7 +75,7 @@ class FlowModelGenerator { const results = decodedResults.tuples; - return results.map((result): [string, ModeledMethod] => { + return results.map((result) => { const row = result[0] as string; return definition.readModeledMethod(row.split(";")); @@ -80,9 +83,7 @@ class FlowModelGenerator { } async run( - onResults: ( - results: Array<[string, ModeledMethod]>, - ) => void | Promise, + onResults: (results: ModeledMethodWithSignature[]) => void | Promise, ) { const summaryResults = await this.getAddsTo( "summary", @@ -124,7 +125,7 @@ export async function generateFlowModel( queryStorageDir: string, qlDir: string, databaseItem: DatabaseItem, - onResults: (results: Array<[string, ModeledMethod]>) => void | Promise, + onResults: (results: ModeledMethodWithSignature[]) => void | Promise, progress: ProgressCallback, token: CancellationToken, ) { diff --git a/extensions/ql-vscode/src/data-extensions-editor/modeled-method.ts b/extensions/ql-vscode/src/data-extensions-editor/modeled-method.ts index cac4fa466..27c3d5f15 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/modeled-method.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/modeled-method.ts @@ -11,3 +11,8 @@ export type ModeledMethod = { output: string; kind: string; }; + +export type ModeledMethodWithSignature = { + signature: string; + modeledMethod: ModeledMethod; +}; diff --git a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts index c573d1f88..95383173d 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/yaml.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/yaml.ts @@ -1,5 +1,9 @@ import { ExternalApiUsage } from "./external-api-usage"; -import { ModeledMethod, ModeledMethodType } from "./modeled-method"; +import { + ModeledMethod, + ModeledMethodType, + ModeledMethodWithSignature, +} from "./modeled-method"; type ExternalApiUsageByType = { externalApiUsage: ExternalApiUsage; @@ -9,7 +13,7 @@ type ExternalApiUsageByType = { type ExtensiblePredicateDefinition = { extensiblePredicate: string; generateMethodDefinition: (method: ExternalApiUsageByType) => any[]; - readModeledMethod: (row: any[]) => [string, ModeledMethod]; + readModeledMethod: (row: any[]) => ModeledMethodWithSignature; }; function readRowToMethod(row: any[]): string { @@ -37,15 +41,15 @@ export const extensiblePredicateDefinitions: Record< method.modeledMethod.kind, "manual", ], - readModeledMethod: (row) => [ - readRowToMethod(row), - { + readModeledMethod: (row) => ({ + signature: readRowToMethod(row), + modeledMethod: { type: "source", input: "", output: row[6], kind: row[7], }, - ], + }), }, sink: { extensiblePredicate: "sinkModel", @@ -64,15 +68,15 @@ export const extensiblePredicateDefinitions: Record< method.modeledMethod.kind, "manual", ], - readModeledMethod: (row) => [ - readRowToMethod(row), - { + readModeledMethod: (row) => ({ + signature: readRowToMethod(row), + modeledMethod: { type: "sink", input: row[6], output: "", kind: row[7], }, - ], + }), }, summary: { extensiblePredicate: "summaryModel", @@ -92,15 +96,15 @@ export const extensiblePredicateDefinitions: Record< method.modeledMethod.kind, "manual", ], - readModeledMethod: (row) => [ - readRowToMethod(row), - { + readModeledMethod: (row) => ({ + signature: readRowToMethod(row), + modeledMethod: { type: "summary", input: row[6], output: row[7], kind: row[8], }, - ], + }), }, neutral: { extensiblePredicate: "neutralModel", @@ -114,15 +118,15 @@ export const extensiblePredicateDefinitions: Record< method.externalApiUsage.methodParameters, "manual", ], - readModeledMethod: (row) => [ - `${row[0]}.${row[1]}#${row[2]}${row[3]}`, - { + readModeledMethod: (row) => ({ + signature: `${row[0]}.${row[1]}#${row[2]}${row[3]}`, + modeledMethod: { type: "neutral", input: "", output: "", kind: "", }, - ], + }), }, }; @@ -227,9 +231,9 @@ export function loadDataExtensionYaml( continue; } - const [apiInfo, modeledMethod] = result; + const { signature, modeledMethod } = result; - modeledMethods[apiInfo] = modeledMethod; + modeledMethods[signature] = modeledMethod; } } From 18db74ed2daa8102a637cd7c317913a24ac45f29 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:35:04 +0200 Subject: [PATCH 17/25] Add type check for query results --- .../data-extensions-editor/generate-flow-model.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index 8f737fe3e..445754d91 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -75,11 +75,16 @@ class FlowModelGenerator { const results = decodedResults.tuples; - return results.map((result) => { - const row = result[0] as string; + return ( + results + // This is just a sanity check. The query should only return strings. + .filter((result) => typeof result[0] === "string") + .map((result) => { + const row = result[0] as string; - return definition.readModeledMethod(row.split(";")); - }); + return definition.readModeledMethod(row.split(";")); + }) + ); } async run( From af8e0bb45478a7ba11750e4cb5bc30133bf06cc1 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:49:20 +0200 Subject: [PATCH 18/25] Rename getAddsTo --- .../data-extensions-editor/generate-flow-model.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index 445754d91..8c3c61013 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -23,7 +23,7 @@ class FlowModelGenerator { private readonly token: CancellationToken, ) {} - async getAddsTo( + private async getModeledMethodsFromFlow( type: Exclude, queryName: string, queryStep: number, @@ -90,7 +90,7 @@ class FlowModelGenerator { async run( onResults: (results: ModeledMethodWithSignature[]) => void | Promise, ) { - const summaryResults = await this.getAddsTo( + const summaryResults = await this.getModeledMethodsFromFlow( "summary", "CaptureSummaryModels.ql", 0, @@ -99,12 +99,16 @@ class FlowModelGenerator { await onResults(summaryResults); } - const sinkResults = await this.getAddsTo("sink", "CaptureSinkModels.ql", 1); + const sinkResults = await this.getModeledMethodsFromFlow( + "sink", + "CaptureSinkModels.ql", + 1, + ); if (sinkResults) { await onResults(sinkResults); } - const sourceResults = await this.getAddsTo( + const sourceResults = await this.getModeledMethodsFromFlow( "source", "CaptureSourceModels.ql", 2, @@ -113,7 +117,7 @@ class FlowModelGenerator { await onResults(sourceResults); } - const neutralResults = await this.getAddsTo( + const neutralResults = await this.getModeledMethodsFromFlow( "neutral", "CaptureNeutralModels.ql", 3, From dbc7f90c205c28cae22d74ebcb110e66063a23ff Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 6 Apr 2023 16:54:20 +0200 Subject: [PATCH 19/25] Throw error instead of returning undefined --- .../data-extensions-editor/data-extensions-editor-view.ts | 6 +++++- .../src/data-extensions-editor/generate-flow-model.ts | 7 +++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 1622f6875..b4da79e8a 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -294,7 +294,11 @@ export class DataExtensionsEditorView extends AbstractWebview< tokenSource.token, ); } catch (e: unknown) { - void extLogger.log(`Error: ${getErrorMessage(e)}`); + void showAndLogExceptionWithTelemetry( + redactableError( + asError(e), + )`Failed to generate flow model: ${getErrorMessage(e)}`, + ); } await this.databaseManager.removeDatabaseItem( diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index 8c3c61013..e60960eb5 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -3,7 +3,7 @@ import { DatabaseItem } from "../local-databases"; import { join } from "path"; import { QueryRunner } from "../queryRunner"; import { CodeQLCliServer } from "../cli"; -import { extLogger, TeeLogger } from "../common"; +import { TeeLogger } from "../common"; import { extensiblePredicateDefinitions } from "./yaml"; import { ProgressCallback } from "../progress"; import { getOnDiskWorkspaceFolders } from "../helpers"; @@ -27,7 +27,7 @@ class FlowModelGenerator { type: Exclude, queryName: string, queryStep: number, - ): Promise { + ): Promise { const definition = extensiblePredicateDefinitions[type]; const query = join( @@ -63,10 +63,9 @@ class FlowModelGenerator { const bqrsInfo = await this.cli.bqrsInfo(bqrsPath); if (bqrsInfo["result-sets"].length !== 1) { - void extLogger.log( + throw new Error( `Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`, ); - return undefined; } const resultSet = bqrsInfo["result-sets"][0]; From f21117e5ddf815f053b9dc6b1d9b4eef3e6f15a1 Mon Sep 17 00:00:00 2001 From: Nora Date: Thu, 6 Apr 2023 13:18:24 +0000 Subject: [PATCH 20/25] Extract read and write to store --- .../variant-analysis/store/repo-task-store.ts | 18 ++++++++++++++++++ .../variant-analysis-results-manager.ts | 13 +++++-------- 2 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 extensions/ql-vscode/src/variant-analysis/store/repo-task-store.ts diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-task-store.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-task-store.ts new file mode 100644 index 000000000..8baab2cb8 --- /dev/null +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-task-store.ts @@ -0,0 +1,18 @@ +import { outputJson, readJson } from "fs-extra"; +import { join } from "path"; +import { VariantAnalysisRepositoryTask } from "../shared/variant-analysis"; + +export const REPO_TASK_FILENAME = "repo_task.json"; + +export function writeRepoTask( + storageDirectory: string, + repoTask: VariantAnalysisRepositoryTask, +): Promise { + return outputJson(join(storageDirectory, REPO_TASK_FILENAME), repoTask); +} + +export function readRepoTask( + storageDirectory: string, +): Promise { + return readJson(join(storageDirectory, REPO_TASK_FILENAME)); +} diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts index 46abb28cb..1c1fa8ace 100644 --- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts +++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts @@ -1,4 +1,4 @@ -import { appendFile, pathExists, mkdir, outputJson, readJson } from "fs-extra"; +import { appendFile, pathExists, mkdir } from "fs-extra"; import fetch from "node-fetch"; import { EOL } from "os"; import { join } from "path"; @@ -17,6 +17,7 @@ import { import { DisposableObject, DisposeHandler } from "../pure/disposable-object"; import { EventEmitter } from "vscode"; import { unzipFile } from "../pure/zip"; +import { readRepoTask, writeRepoTask } from "./store/repo-task-store"; type CacheKey = `${number}/${string}`; @@ -37,7 +38,6 @@ export type LoadResultsOptions = { }; export class VariantAnalysisResultsManager extends DisposableObject { - private static readonly REPO_TASK_FILENAME = "repo_task.json"; private static readonly RESULTS_DIRECTORY = "results"; private readonly cachedResults: Map< @@ -82,10 +82,7 @@ export class VariantAnalysisResultsManager extends DisposableObject { await mkdir(resultDirectory, { recursive: true }); } - await outputJson( - join(resultDirectory, VariantAnalysisResultsManager.REPO_TASK_FILENAME), - repoTask, - ); + await writeRepoTask(resultDirectory, repoTask); const zipFilePath = join(resultDirectory, "results.zip"); @@ -184,8 +181,8 @@ export class VariantAnalysisResultsManager extends DisposableObject { repositoryFullName, ); - const repoTask: VariantAnalysisRepositoryTask = await readJson( - join(storageDirectory, VariantAnalysisResultsManager.REPO_TASK_FILENAME), + const repoTask: VariantAnalysisRepositoryTask = await readRepoTask( + storageDirectory, ); if (!repoTask.databaseCommitSha || !repoTask.sourceLocationPrefix) { From 1beddf76403b6cd9b8ffd3a084433e521c34612d Mon Sep 17 00:00:00 2001 From: Nora Date: Thu, 6 Apr 2023 14:50:54 +0000 Subject: [PATCH 21/25] Map data types --- .../store/repo-task-data-types.ts | 25 ++++++++++ .../variant-analysis/store/repo-task-store.ts | 17 +++++-- .../store/repo-task-to-data-mapper.ts | 49 +++++++++++++++++++ .../store/repo-task-to-domain-mapper.ts | 49 +++++++++++++++++++ 4 files changed, 136 insertions(+), 4 deletions(-) create mode 100644 extensions/ql-vscode/src/variant-analysis/store/repo-task-data-types.ts create mode 100644 extensions/ql-vscode/src/variant-analysis/store/repo-task-to-data-mapper.ts create mode 100644 extensions/ql-vscode/src/variant-analysis/store/repo-task-to-domain-mapper.ts diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-task-data-types.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-task-data-types.ts new file mode 100644 index 000000000..22dbef63b --- /dev/null +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-task-data-types.ts @@ -0,0 +1,25 @@ +export interface VariantAnalysisRepositoryTaskData { + repository: RepositoryData; + analysisStatus: VariantAnalysisRepoStatusData; + resultCount?: number; + artifactSizeInBytes?: number; + failureMessage?: string; + databaseCommitSha?: string; + sourceLocationPrefix?: string; + artifactUrl?: string; +} + +interface RepositoryData { + id: number; + fullName: string; + private: boolean; +} + +export enum VariantAnalysisRepoStatusData { + Pending = "pending", + InProgress = "inProgress", + Succeeded = "succeeded", + Failed = "failed", + Canceled = "canceled", + TimedOut = "timedOut", +} diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-task-store.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-task-store.ts index 8baab2cb8..9121164ca 100644 --- a/extensions/ql-vscode/src/variant-analysis/store/repo-task-store.ts +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-task-store.ts @@ -1,18 +1,27 @@ import { outputJson, readJson } from "fs-extra"; import { join } from "path"; import { VariantAnalysisRepositoryTask } from "../shared/variant-analysis"; +import { mapRepoTaskToData } from "./repo-task-to-data-mapper"; +import { mapRepoTaskToDomain } from "./repo-task-to-domain-mapper"; export const REPO_TASK_FILENAME = "repo_task.json"; -export function writeRepoTask( +export async function writeRepoTask( storageDirectory: string, repoTask: VariantAnalysisRepositoryTask, ): Promise { - return outputJson(join(storageDirectory, REPO_TASK_FILENAME), repoTask); + const repoTaskData = mapRepoTaskToData(repoTask); + return await outputJson( + join(storageDirectory, REPO_TASK_FILENAME), + repoTaskData, + ); } -export function readRepoTask( +export async function readRepoTask( storageDirectory: string, ): Promise { - return readJson(join(storageDirectory, REPO_TASK_FILENAME)); + const repoTaskData = await readJson( + join(storageDirectory, REPO_TASK_FILENAME), + ); + return mapRepoTaskToDomain(repoTaskData); } diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-task-to-data-mapper.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-task-to-data-mapper.ts new file mode 100644 index 000000000..1fc3a1a7b --- /dev/null +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-task-to-data-mapper.ts @@ -0,0 +1,49 @@ +import { assertNever } from "../../pure/helpers-pure"; +import { + VariantAnalysisRepositoryTask, + VariantAnalysisRepoStatus, +} from "../shared/variant-analysis"; +import { + VariantAnalysisRepositoryTaskData, + VariantAnalysisRepoStatusData, +} from "./repo-task-data-types"; + +export function mapRepoTaskToData( + repoTask: VariantAnalysisRepositoryTask, +): VariantAnalysisRepositoryTaskData { + return { + repository: { + id: repoTask.repository.id, + fullName: repoTask.repository.fullName, + private: repoTask.repository.private, + }, + analysisStatus: mapRepoTaskAnalysisStatusToData(repoTask.analysisStatus), + resultCount: repoTask.resultCount, + artifactSizeInBytes: repoTask.artifactSizeInBytes, + failureMessage: repoTask.failureMessage, + databaseCommitSha: repoTask.databaseCommitSha, + sourceLocationPrefix: repoTask.sourceLocationPrefix, + artifactUrl: repoTask.artifactUrl, + }; +} + +function mapRepoTaskAnalysisStatusToData( + analysisStatus: VariantAnalysisRepoStatus, +): VariantAnalysisRepoStatusData { + switch (analysisStatus) { + case VariantAnalysisRepoStatus.Pending: + return VariantAnalysisRepoStatusData.Pending; + case VariantAnalysisRepoStatus.InProgress: + return VariantAnalysisRepoStatusData.InProgress; + case VariantAnalysisRepoStatus.Succeeded: + return VariantAnalysisRepoStatusData.Succeeded; + case VariantAnalysisRepoStatus.Failed: + return VariantAnalysisRepoStatusData.Failed; + case VariantAnalysisRepoStatus.Canceled: + return VariantAnalysisRepoStatusData.Canceled; + case VariantAnalysisRepoStatus.TimedOut: + return VariantAnalysisRepoStatusData.TimedOut; + default: + assertNever(analysisStatus); + } +} diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-task-to-domain-mapper.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-task-to-domain-mapper.ts new file mode 100644 index 000000000..341eaba8a --- /dev/null +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-task-to-domain-mapper.ts @@ -0,0 +1,49 @@ +import { assertNever } from "../../pure/helpers-pure"; +import { + VariantAnalysisRepositoryTask, + VariantAnalysisRepoStatus, +} from "../shared/variant-analysis"; +import { + VariantAnalysisRepositoryTaskData, + VariantAnalysisRepoStatusData, +} from "./repo-task-data-types"; + +export function mapRepoTaskToDomain( + repoTask: VariantAnalysisRepositoryTaskData, +): VariantAnalysisRepositoryTask { + return { + repository: { + id: repoTask.repository.id, + fullName: repoTask.repository.fullName, + private: repoTask.repository.private, + }, + analysisStatus: mapRepoTaskAnalysisStatusToDomain(repoTask.analysisStatus), + resultCount: repoTask.resultCount, + artifactSizeInBytes: repoTask.artifactSizeInBytes, + failureMessage: repoTask.failureMessage, + databaseCommitSha: repoTask.databaseCommitSha, + sourceLocationPrefix: repoTask.sourceLocationPrefix, + artifactUrl: repoTask.artifactUrl, + }; +} + +function mapRepoTaskAnalysisStatusToDomain( + analysisStatus: VariantAnalysisRepoStatusData, +): VariantAnalysisRepoStatus { + switch (analysisStatus) { + case VariantAnalysisRepoStatusData.Pending: + return VariantAnalysisRepoStatus.Pending; + case VariantAnalysisRepoStatusData.InProgress: + return VariantAnalysisRepoStatus.InProgress; + case VariantAnalysisRepoStatusData.Succeeded: + return VariantAnalysisRepoStatus.Succeeded; + case VariantAnalysisRepoStatusData.Failed: + return VariantAnalysisRepoStatus.Failed; + case VariantAnalysisRepoStatusData.Canceled: + return VariantAnalysisRepoStatus.Canceled; + case VariantAnalysisRepoStatusData.TimedOut: + return VariantAnalysisRepoStatus.TimedOut; + default: + assertNever(analysisStatus); + } +} From 2963d7eb5f47ebbd9e837d4846adc6a987a45384 Mon Sep 17 00:00:00 2001 From: Nora Date: Thu, 6 Apr 2023 14:51:57 +0000 Subject: [PATCH 22/25] Remove unnecessary return statements in repo task and repo states --- .../src/variant-analysis/store/repo-states-store.ts | 2 +- .../ql-vscode/src/variant-analysis/store/repo-task-store.ts | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts index ce517f4ce..1477e4387 100644 --- a/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-states-store.ts @@ -17,7 +17,7 @@ export async function writeRepoStates( }), ); - return await outputJson(storagePath, repoStatesData); + await outputJson(storagePath, repoStatesData); } export async function readRepoStates( diff --git a/extensions/ql-vscode/src/variant-analysis/store/repo-task-store.ts b/extensions/ql-vscode/src/variant-analysis/store/repo-task-store.ts index 9121164ca..0c9152a2b 100644 --- a/extensions/ql-vscode/src/variant-analysis/store/repo-task-store.ts +++ b/extensions/ql-vscode/src/variant-analysis/store/repo-task-store.ts @@ -11,10 +11,7 @@ export async function writeRepoTask( repoTask: VariantAnalysisRepositoryTask, ): Promise { const repoTaskData = mapRepoTaskToData(repoTask); - return await outputJson( - join(storageDirectory, REPO_TASK_FILENAME), - repoTaskData, - ); + await outputJson(join(storageDirectory, REPO_TASK_FILENAME), repoTaskData); } export async function readRepoTask( From 2cb5928cd84b68225d1aae10afe7e35be4bdf436 Mon Sep 17 00:00:00 2001 From: Nora Date: Thu, 6 Apr 2023 14:57:28 +0000 Subject: [PATCH 23/25] Remove unnecessary directory creation that is handled by outputJson --- .../variant-analysis/variant-analysis-results-manager.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts index 1c1fa8ace..2f5691ac3 100644 --- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts +++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts @@ -1,4 +1,4 @@ -import { appendFile, pathExists, mkdir } from "fs-extra"; +import { appendFile, pathExists } from "fs-extra"; import fetch from "node-fetch"; import { EOL } from "os"; import { join } from "path"; @@ -78,10 +78,6 @@ export class VariantAnalysisResultsManager extends DisposableObject { repoTask.repository.fullName, ); - if (!(await pathExists(resultDirectory))) { - await mkdir(resultDirectory, { recursive: true }); - } - await writeRepoTask(resultDirectory, repoTask); const zipFilePath = join(resultDirectory, "results.zip"); From 4e8df309fb5d231ccf3f0fed0fb04a6b442f2d1a Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Tue, 11 Apr 2023 10:22:10 +0200 Subject: [PATCH 24/25] Add comments for adding flow database --- .../data-extensions-editor/data-extensions-editor-view.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index b4da79e8a..5e7be0a45 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -241,6 +241,9 @@ export class DataExtensionsEditorView extends AbstractWebview< const selectedDatabase = this.databaseManager.currentDatabaseItem; + // The external API methods are in the library source code, so we need to ask + // the user to import the library database. We need to have the database + // imported to the query server, so we need to register it to our workspace. const database = await promptImportGithubDatabase( this.app.commands, this.databaseManager, @@ -257,6 +260,8 @@ export class DataExtensionsEditorView extends AbstractWebview< return; } + // The library database was set as the current database by importing it, + // but we need to set it back to the originally selected database. await this.databaseManager.setCurrentDatabaseItem(selectedDatabase); const workspaceFolder = getQlSubmoduleFolder(); @@ -301,6 +306,8 @@ export class DataExtensionsEditorView extends AbstractWebview< ); } + // After the flow model has been generated, we can remove the temporary database + // which we used for generating the flow model. await this.databaseManager.removeDatabaseItem( () => this.showProgress({ From 102976e167b804dd205629fc4c11439383fc68a4 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Tue, 11 Apr 2023 10:27:46 +0200 Subject: [PATCH 25/25] Use functions instead of class for generating flow model We were using a single-use class for generating the flow model, while we are actually able to do it using two functions. This is more in line with our existing codebase. --- .../data-extensions-editor-view.ts | 20 +- .../generate-flow-model.ts | 244 +++++++++--------- 2 files changed, 127 insertions(+), 137 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 5e7be0a45..7975c884b 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -276,13 +276,13 @@ export class DataExtensionsEditorView extends AbstractWebview< }); try { - await generateFlowModel( - this.cliServer, - this.queryRunner, - this.queryStorageDir, - workspaceFolder.uri.fsPath, - database, - async (results) => { + await generateFlowModel({ + cliServer: this.cliServer, + queryRunner: this.queryRunner, + queryStorageDir: this.queryStorageDir, + qlDir: workspaceFolder.uri.fsPath, + databaseItem: database, + onResults: async (results) => { const modeledMethodsByName: Record = {}; for (const result of results) { @@ -295,9 +295,9 @@ export class DataExtensionsEditorView extends AbstractWebview< overrideNone: true, }); }, - (update) => this.showProgress(update), - tokenSource.token, - ); + progress: (update) => this.showProgress(update), + token: tokenSource.token, + }); } catch (e: unknown) { void showAndLogExceptionWithTelemetry( redactableError( diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts index e60960eb5..f6125d0f3 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts @@ -12,140 +12,130 @@ import { ModeledMethodWithSignature, } from "./modeled-method"; -class FlowModelGenerator { - constructor( - private readonly cli: CodeQLCliServer, - private readonly queryRunner: QueryRunner, - private readonly queryStorageDir: string, - private readonly qlDir: string, - private readonly databaseItem: DatabaseItem, - private readonly progress: ProgressCallback, - private readonly token: CancellationToken, - ) {} +type FlowModelOptions = { + cliServer: CodeQLCliServer; + queryRunner: QueryRunner; + queryStorageDir: string; + qlDir: string; + databaseItem: DatabaseItem; + progress: ProgressCallback; + token: CancellationToken; + onResults: (results: ModeledMethodWithSignature[]) => void | Promise; +}; - private async getModeledMethodsFromFlow( - type: Exclude, - queryName: string, - queryStep: number, - ): Promise { - const definition = extensiblePredicateDefinitions[type]; - - const query = join( - this.qlDir, - this.databaseItem.language, - "ql/src/utils/modelgenerator", - queryName, - ); - - const queryRun = this.queryRunner.createQueryRun( - this.databaseItem.databaseUri.fsPath, - { queryPath: query, quickEvalPosition: undefined }, - false, - getOnDiskWorkspaceFolders(), - undefined, - this.queryStorageDir, - undefined, - undefined, - ); - - const queryResult = await queryRun.evaluate( - ({ step, message }) => - this.progress({ - message: `Generating ${type} model: ${message}`, - step: queryStep * 1000 + step, - maxStep: 4000, - }), - this.token, - new TeeLogger(this.queryRunner.logger, queryRun.outputDir.logPath), - ); - - const bqrsPath = queryResult.outputDir.bqrsPath; - - const bqrsInfo = await this.cli.bqrsInfo(bqrsPath); - if (bqrsInfo["result-sets"].length !== 1) { - throw new Error( - `Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`, - ); - } - - const resultSet = bqrsInfo["result-sets"][0]; - - const decodedResults = await this.cli.bqrsDecode(bqrsPath, resultSet.name); - - const results = decodedResults.tuples; - - return ( - results - // This is just a sanity check. The query should only return strings. - .filter((result) => typeof result[0] === "string") - .map((result) => { - const row = result[0] as string; - - return definition.readModeledMethod(row.split(";")); - }) - ); - } - - async run( - onResults: (results: ModeledMethodWithSignature[]) => void | Promise, - ) { - const summaryResults = await this.getModeledMethodsFromFlow( - "summary", - "CaptureSummaryModels.ql", - 0, - ); - if (summaryResults) { - await onResults(summaryResults); - } - - const sinkResults = await this.getModeledMethodsFromFlow( - "sink", - "CaptureSinkModels.ql", - 1, - ); - if (sinkResults) { - await onResults(sinkResults); - } - - const sourceResults = await this.getModeledMethodsFromFlow( - "source", - "CaptureSourceModels.ql", - 2, - ); - if (sourceResults) { - await onResults(sourceResults); - } - - const neutralResults = await this.getModeledMethodsFromFlow( - "neutral", - "CaptureNeutralModels.ql", - 3, - ); - if (neutralResults) { - await onResults(neutralResults); - } - } -} - -export async function generateFlowModel( - cli: CodeQLCliServer, - queryRunner: QueryRunner, - queryStorageDir: string, - qlDir: string, - databaseItem: DatabaseItem, - onResults: (results: ModeledMethodWithSignature[]) => void | Promise, - progress: ProgressCallback, - token: CancellationToken, -) { - const generator = new FlowModelGenerator( - cli, +async function getModeledMethodsFromFlow( + type: Exclude, + queryName: string, + queryStep: number, + { + cliServer, queryRunner, queryStorageDir, qlDir, databaseItem, progress, token, + }: Omit, +): Promise { + const definition = extensiblePredicateDefinitions[type]; + + const query = join( + qlDir, + databaseItem.language, + "ql/src/utils/modelgenerator", + queryName, ); - return generator.run(onResults); + const queryRun = queryRunner.createQueryRun( + databaseItem.databaseUri.fsPath, + { queryPath: query, quickEvalPosition: undefined }, + false, + getOnDiskWorkspaceFolders(), + undefined, + queryStorageDir, + undefined, + undefined, + ); + + const queryResult = await queryRun.evaluate( + ({ step, message }) => + progress({ + message: `Generating ${type} model: ${message}`, + step: queryStep * 1000 + step, + maxStep: 4000, + }), + token, + new TeeLogger(queryRunner.logger, queryRun.outputDir.logPath), + ); + + const bqrsPath = queryResult.outputDir.bqrsPath; + + const bqrsInfo = await cliServer.bqrsInfo(bqrsPath); + if (bqrsInfo["result-sets"].length !== 1) { + throw new Error( + `Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`, + ); + } + + const resultSet = bqrsInfo["result-sets"][0]; + + const decodedResults = await cliServer.bqrsDecode(bqrsPath, resultSet.name); + + const results = decodedResults.tuples; + + return ( + results + // This is just a sanity check. The query should only return strings. + .filter((result) => typeof result[0] === "string") + .map((result) => { + const row = result[0] as string; + + return definition.readModeledMethod(row.split(";")); + }) + ); +} + +export async function generateFlowModel({ + onResults, + ...options +}: FlowModelOptions) { + const summaryResults = await getModeledMethodsFromFlow( + "summary", + "CaptureSummaryModels.ql", + 0, + options, + ); + if (summaryResults) { + await onResults(summaryResults); + } + + const sinkResults = await getModeledMethodsFromFlow( + "sink", + "CaptureSinkModels.ql", + 1, + options, + ); + if (sinkResults) { + await onResults(sinkResults); + } + + const sourceResults = await getModeledMethodsFromFlow( + "source", + "CaptureSourceModels.ql", + 2, + options, + ); + if (sourceResults) { + await onResults(sourceResults); + } + + const neutralResults = await getModeledMethodsFromFlow( + "neutral", + "CaptureNeutralModels.ql", + 3, + options, + ); + if (neutralResults) { + await onResults(neutralResults); + } }