diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts index 61a29ea4d..2cbf6a53d 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts @@ -1,14 +1,36 @@ import { ExtensionContext } from "vscode"; import { DataExtensionsEditorView } from "./data-extensions-editor-view"; import { DataExtensionsEditorCommands } from "../common/commands"; +import { CodeQLCliServer } from "../cli"; +import { QueryRunner } from "../queryRunner"; +import { DatabaseManager } from "../local-databases"; +import { extLogger } from "../common"; export class DataExtensionsEditorModule { - public constructor(private readonly ctx: ExtensionContext) {} + public constructor( + private readonly ctx: ExtensionContext, + private readonly databaseManager: DatabaseManager, + private readonly cliServer: CodeQLCliServer, + private readonly queryRunner: QueryRunner, + private readonly queryStorageDir: string, + ) {} public getCommands(): DataExtensionsEditorCommands { return { "codeQL.openDataExtensionsEditor": async () => { - const view = new DataExtensionsEditorView(this.ctx); + const db = this.databaseManager.currentDatabaseItem; + if (!db) { + void extLogger.log("No database selected"); + return; + } + + const view = new DataExtensionsEditorView( + this.ctx, + this.cliServer, + this.queryRunner, + this.queryStorageDir, + db, + ); await view.openView(); }, }; diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index d8456ccb0..9b0696ead 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -1,15 +1,31 @@ -import { ExtensionContext, ViewColumn } from "vscode"; +import { CancellationTokenSource, ExtensionContext, ViewColumn } from "vscode"; import { AbstractWebview, WebviewPanelConfig } from "../abstract-webview"; import { FromDataExtensionsEditorMessage, ToDataExtensionsEditorMessage, } from "../pure/interface-types"; +import { ProgressUpdate } from "../progress"; +import { extLogger, TeeLogger } from "../common"; +import { CoreCompletedQuery, QueryRunner } from "../queryRunner"; +import { qlpackOfDatabase } from "../contextual/queryResolver"; +import { file } from "tmp-promise"; +import { writeFile } from "fs-extra"; +import { dump } from "js-yaml"; +import { getOnDiskWorkspaceFolders } from "../helpers"; +import { DatabaseItem } from "../local-databases"; +import { CodeQLCliServer } from "../cli"; export class DataExtensionsEditorView extends AbstractWebview< ToDataExtensionsEditorMessage, FromDataExtensionsEditorMessage > { - public constructor(ctx: ExtensionContext) { + public constructor( + ctx: ExtensionContext, + private readonly cliServer: CodeQLCliServer, + private readonly queryRunner: QueryRunner, + private readonly queryStorageDir: string, + private readonly databaseItem: DatabaseItem, + ) { super(ctx); } @@ -49,5 +65,136 @@ export class DataExtensionsEditorView extends AbstractWebview< protected async onWebViewLoaded() { super.onWebViewLoaded(); + + await this.loadExternalApiUsages(); + } + + protected async loadExternalApiUsages(): Promise { + const queryResult = await this.runQuery(); + if (!queryResult) { + await this.clearProgress(); + return; + } + + await this.showProgress({ + message: "Loading results", + step: 1100, + maxStep: 1500, + }); + + const bqrsPath = queryResult.outputDir.bqrsPath; + + const results = await this.getResults(bqrsPath); + if (!results) { + await this.clearProgress(); + return; + } + + await this.showProgress({ + message: "Finalizing results", + step: 1450, + maxStep: 1500, + }); + + await this.postMessage({ + t: "setExternalApiRepoResults", + results, + }); + + await this.clearProgress(); + } + + private async runQuery(): Promise { + const qlpacks = await qlpackOfDatabase(this.cliServer, this.databaseItem); + + const packsToSearch = [qlpacks.dbschemePack]; + if (qlpacks.queryPack) { + packsToSearch.push(qlpacks.queryPack); + } + + const suiteFile = ( + await file({ + postfix: ".qls", + }) + ).path; + const suiteYaml = []; + for (const qlpack of packsToSearch) { + suiteYaml.push({ + from: qlpack, + queries: ".", + include: { + id: `${this.databaseItem.language}/telemetry/fetch-external-apis`, + }, + }); + } + await writeFile(suiteFile, dump(suiteYaml), "utf8"); + + const queries = await this.cliServer.resolveQueriesInSuite( + suiteFile, + getOnDiskWorkspaceFolders(), + ); + + if (queries.length !== 1) { + void extLogger.log(`Expected exactly one query, got ${queries.length}`); + return; + } + + const query = queries[0]; + + const tokenSource = new CancellationTokenSource(); + + const queryRun = this.queryRunner.createQueryRun( + this.databaseItem.databaseUri.fsPath, + { queryPath: query, quickEvalPosition: undefined }, + false, + getOnDiskWorkspaceFolders(), + undefined, + this.queryStorageDir, + undefined, + undefined, + ); + + return queryRun.evaluate( + (update) => this.showProgress(update, 1500), + tokenSource.token, + new TeeLogger(this.queryRunner.logger, queryRun.outputDir.logPath), + ); + } + + private async getResults(bqrsPath: string) { + const bqrsInfo = await this.cliServer.bqrsInfo(bqrsPath); + if (bqrsInfo["result-sets"].length !== 1) { + void extLogger.log( + `Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`, + ); + return undefined; + } + + const resultSet = bqrsInfo["result-sets"][0]; + + await this.showProgress({ + message: "Decoding results", + step: 1200, + maxStep: 1500, + }); + + return this.cliServer.bqrsDecode(bqrsPath, resultSet.name); + } + + private async showProgress(update: ProgressUpdate, maxStep?: number) { + await this.postMessage({ + t: "showProgress", + step: update.step, + maxStep: maxStep ?? update.maxStep, + message: update.message, + }); + } + + private async clearProgress() { + await this.showProgress({ + step: 0, + maxStep: 0, + message: "", + }); } } diff --git a/extensions/ql-vscode/src/data-extensions-editor/interface.ts b/extensions/ql-vscode/src/data-extensions-editor/interface.ts new file mode 100644 index 000000000..355a4963e --- /dev/null +++ b/extensions/ql-vscode/src/data-extensions-editor/interface.ts @@ -0,0 +1,30 @@ +import { ResolvableLocationValue } from "../pure/bqrs-cli-types"; + +export type Call = { + label: string; + url: ResolvableLocationValue; +}; + +export type ExternalApiUsage = { + externalApiInfo: string; + packageName: string; + typeName: string; + methodName: string; + methodParameters: string; + supported: boolean; + usages: Call[]; +}; + +export type ModeledMethodType = + | "none" + | "source" + | "sink" + | "summary" + | "neutral"; + +export type ModeledMethod = { + type: ModeledMethodType; + input: string; + output: string; + kind: string; +}; diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index 391acd501..db25f3195 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -861,7 +861,18 @@ async function activateWithInstalledDistribution( ); ctx.subscriptions.push(localQueries); - const dataExtensionsEditorModule = new DataExtensionsEditorModule(ctx); + const dataExtensionsEditorQueryStorageDir = join( + tmpDir.name, + "data-extensions-editor-results", + ); + await ensureDir(dataExtensionsEditorQueryStorageDir); + const dataExtensionsEditorModule = new DataExtensionsEditorModule( + ctx, + dbm, + cliServer, + qs, + dataExtensionsEditorQueryStorageDir, + ); void extLogger.log("Initializing QLTest interface."); const testExplorerExtension = extensions.getExtension( diff --git a/extensions/ql-vscode/src/pure/interface-types.ts b/extensions/ql-vscode/src/pure/interface-types.ts index 95d9475e5..b3ed4809c 100644 --- a/extensions/ql-vscode/src/pure/interface-types.ts +++ b/extensions/ql-vscode/src/pure/interface-types.ts @@ -5,6 +5,7 @@ import { ResultSetSchema, Column, ResolvableLocationValue, + DecodedBqrsChunk, } from "./bqrs-cli-types"; import { VariantAnalysis, @@ -479,6 +480,20 @@ export type ToDataFlowPathsMessage = SetDataFlowPathsMessage; export type FromDataFlowPathsMessage = CommonFromViewMessages; -export type ToDataExtensionsEditorMessage = never; +export interface SetExternalApiResultsMessage { + t: "setExternalApiRepoResults"; + results: DecodedBqrsChunk; +} + +export interface ShowProgressMessage { + t: "showProgress"; + step: number; + maxStep: number; + message: string; +} + +export type ToDataExtensionsEditorMessage = + | SetExternalApiResultsMessage + | ShowProgressMessage; export type FromDataExtensionsEditorMessage = ViewLoadedMsg; diff --git a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx index 0edc98a66..2cd4515ae 100644 --- a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx +++ b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx @@ -1,5 +1,203 @@ import * as React from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import { DecodedBqrsChunk } from "../../pure/bqrs-cli-types"; +import { + ShowProgressMessage, + ToDataExtensionsEditorMessage, +} from "../../pure/interface-types"; +import { + VSCodeDataGrid, + VSCodeDataGridCell, + VSCodeDataGridRow, +} from "@vscode/webview-ui-toolkit/react"; +import styled from "styled-components"; +import { + Call, + ExternalApiUsage, + ModeledMethod, +} from "../../data-extensions-editor/interface"; +import { MethodRow } from "./MethodRow"; +import { assertNever } from "../../pure/helpers-pure"; + +export const DataExtensionsEditorContainer = styled.div` + margin-top: 1rem; +`; + +type ProgressBarProps = { + completion: number; +}; + +const ProgressBar = styled.div` + height: 10px; + width: ${(props) => props.completion * 100}%; + + background-color: var(--vscode-progressBar-background); +`; export function DataExtensionsEditor(): JSX.Element { - return
Data extensions editor
; + const [results, setResults] = useState( + undefined, + ); + const [modeledMethods, setModeledMethods] = useState< + Record + >({}); + const [progress, setProgress] = useState>({ + step: 0, + maxStep: 0, + message: "", + }); + + useEffect(() => { + const listener = (evt: MessageEvent) => { + if (evt.origin === window.origin) { + const msg: ToDataExtensionsEditorMessage = evt.data; + switch (msg.t) { + case "setExternalApiRepoResults": + setResults(msg.results); + break; + case "showProgress": + setProgress(msg); + break; + default: + assertNever(msg); + } + } else { + // sanitize origin + const origin = evt.origin.replace(/\n|\r/g, ""); + console.error(`Invalid event origin ${origin}`); + } + }; + window.addEventListener("message", listener); + + return () => { + window.removeEventListener("message", listener); + }; + }, []); + + const methods = useMemo(() => { + const methodsByApiName = new Map(); + + results?.tuples.forEach((tuple) => { + const externalApiInfo = tuple[0] as string; + const supported = tuple[1] as boolean; + const usage = tuple[2] as Call; + + const [packageWithType, methodDeclaration] = externalApiInfo.split("#"); + + const packageName = packageWithType.substring( + 0, + packageWithType.lastIndexOf("."), + ); + const typeName = packageWithType.substring( + packageWithType.lastIndexOf(".") + 1, + ); + + const methodName = methodDeclaration.substring( + 0, + methodDeclaration.indexOf("("), + ); + const methodParameters = methodDeclaration.substring( + methodDeclaration.indexOf("("), + ); + + if (!methodsByApiName.has(externalApiInfo)) { + methodsByApiName.set(externalApiInfo, { + externalApiInfo, + packageName, + typeName, + methodName, + methodParameters, + supported, + usages: [], + }); + } + + const method = methodsByApiName.get(externalApiInfo)!; + method.usages.push(usage); + }); + + const externalApiUsages = Array.from(methodsByApiName.values()); + externalApiUsages.sort((a, b) => { + // Sort by number of usages descending + return b.usages.length - a.usages.length; + }); + return externalApiUsages; + }, [results]); + + const supportedPercentage = useMemo(() => { + return (methods.filter((m) => m.supported).length / methods.length) * 100; + }, [methods]); + + const unsupportedPercentage = useMemo(() => { + return (methods.filter((m) => !m.supported).length / methods.length) * 100; + }, [methods]); + + const onChange = useCallback( + (method: ExternalApiUsage, model: ModeledMethod) => { + setModeledMethods((oldModeledMethods) => ({ + ...oldModeledMethods, + [method.externalApiInfo]: model, + })); + }, + [], + ); + + return ( + + {progress.maxStep > 0 && ( +

+ {" "} + {progress.message} +

+ )} + + {methods.length > 0 && ( + <> +
+

External API support stats

+
    +
  • Supported: {supportedPercentage.toFixed(2)}%
    • +
  • Unsupported: {unsupportedPercentage.toFixed(2)}%
    • +
+
+
+

External API modelling

+ + + + Type + + + Method + + + Usages + + + Model type + + + Input + + + Output + + + Kind + + + {methods.map((method) => ( + + ))} + +
+ + )} + + ); } diff --git a/extensions/ql-vscode/src/view/data-extensions-editor/MethodRow.tsx b/extensions/ql-vscode/src/view/data-extensions-editor/MethodRow.tsx new file mode 100644 index 000000000..d02259a20 --- /dev/null +++ b/extensions/ql-vscode/src/view/data-extensions-editor/MethodRow.tsx @@ -0,0 +1,167 @@ +import { + ExternalApiUsage, + ModeledMethod, +} from "../../data-extensions-editor/interface"; +import { + VSCodeDataGridCell, + VSCodeDataGridRow, + VSCodeDropdown, + VSCodeOption, + VSCodeTextField, +} from "@vscode/webview-ui-toolkit/react"; +import * as React from "react"; +import { useCallback, useMemo } from "react"; +import styled from "styled-components"; + +const Dropdown = styled(VSCodeDropdown)` + width: 100%; +`; + +const TextField = styled(VSCodeTextField)` + width: 100%; +`; + +type SupportedUnsupportedSpanProps = { + supported: boolean; +}; + +const SupportedUnsupportedSpan = styled.span` + color: ${(props) => (props.supported ? "green" : "red")}; +`; + +type Props = { + method: ExternalApiUsage; + model: ModeledMethod | undefined; + onChange: (method: ExternalApiUsage, model: ModeledMethod) => void; +}; + +export const MethodRow = ({ method, model, onChange }: Props) => { + const argumentsList = useMemo(() => { + if (method.methodParameters === "()") { + return []; + } + return method.methodParameters + .substring(1, method.methodParameters.length - 1) + .split(","); + }, [method.methodParameters]); + + const handleTypeInput = useCallback( + (e: InputEvent) => { + const target = e.target as HTMLSelectElement; + + onChange(method, { + input: argumentsList.length === 0 ? "Argument[-1]" : "Argument[0]", + output: "ReturnType", + kind: "value", + ...model, + type: target.value as ModeledMethod["type"], + }); + }, + [onChange, method, model, argumentsList], + ); + const handleInputInput = useCallback( + (e: InputEvent) => { + if (!model) { + return; + } + + const target = e.target as HTMLSelectElement; + + onChange(method, { + ...model, + input: target.value as ModeledMethod["input"], + }); + }, + [onChange, method, model], + ); + const handleOutputInput = useCallback( + (e: InputEvent) => { + if (!model) { + return; + } + + const target = e.target as HTMLSelectElement; + + onChange(method, { + ...model, + output: target.value as ModeledMethod["output"], + }); + }, + [onChange, method, model], + ); + const handleKindInput = useCallback( + (e: InputEvent) => { + if (!model) { + return; + } + + const target = e.target as HTMLSelectElement; + + onChange(method, { + ...model, + kind: target.value as ModeledMethod["kind"], + }); + }, + [onChange, method, model], + ); + + return ( + + + + {method.packageName}.{method.typeName} + + + + + {method.methodName} + {method.methodParameters} + + + + {method.usages.length} + + + {(!method.supported || (model && model?.type !== "none")) && ( + + Unmodelled + Source + Sink + Flow summary + Neutral + + )} + + + {model?.type && ["sink", "summary"].includes(model?.type) && ( + + Argument[-1]: this + {argumentsList.map((argument, index) => ( + + Argument[{index}]: {argument} + + ))} + + )} + + + {model?.type && ["source", "summary"].includes(model?.type) && ( + + ReturnValue + Argument[-1]: this + {argumentsList.map((argument, index) => ( + + Argument[{index}]: {argument} + + ))} + + )} + + + {model?.type && ["source", "sink", "summary"].includes(model?.type) && ( + + )} + + + ); +};