Show external API calls in data extensions editor

This updates the view of the data extensions editor to show a table of
possible sources/sinks/flow summaries that can be edited. It's not yet
possible to save the changes or load the existing file.
This commit is contained in:
Koen Vlaswinkel
2023-04-04 13:17:16 +02:00
parent 60e39636e7
commit 73bd6d696c
7 changed files with 597 additions and 7 deletions

View File

@@ -1,14 +1,36 @@
import { ExtensionContext } from "vscode";
import { DataExtensionsEditorView } from "./data-extensions-editor-view";
import { DataExtensionsEditorCommands } from "../common/commands";
import { CodeQLCliServer } from "../cli";
import { QueryRunner } from "../queryRunner";
import { DatabaseManager } from "../local-databases";
import { extLogger } from "../common";
export class DataExtensionsEditorModule {
public constructor(private readonly ctx: ExtensionContext) {}
public constructor(
private readonly ctx: ExtensionContext,
private readonly databaseManager: DatabaseManager,
private readonly cliServer: CodeQLCliServer,
private readonly queryRunner: QueryRunner,
private readonly queryStorageDir: string,
) {}
public getCommands(): DataExtensionsEditorCommands {
return {
"codeQL.openDataExtensionsEditor": async () => {
const view = new DataExtensionsEditorView(this.ctx);
const db = this.databaseManager.currentDatabaseItem;
if (!db) {
void extLogger.log("No database selected");
return;
}
const view = new DataExtensionsEditorView(
this.ctx,
this.cliServer,
this.queryRunner,
this.queryStorageDir,
db,
);
await view.openView();
},
};

View File

@@ -1,15 +1,31 @@
import { ExtensionContext, ViewColumn } from "vscode";
import { CancellationTokenSource, ExtensionContext, ViewColumn } from "vscode";
import { AbstractWebview, WebviewPanelConfig } from "../abstract-webview";
import {
FromDataExtensionsEditorMessage,
ToDataExtensionsEditorMessage,
} from "../pure/interface-types";
import { ProgressUpdate } from "../progress";
import { extLogger, TeeLogger } from "../common";
import { CoreCompletedQuery, QueryRunner } from "../queryRunner";
import { qlpackOfDatabase } from "../contextual/queryResolver";
import { file } from "tmp-promise";
import { writeFile } from "fs-extra";
import { dump } from "js-yaml";
import { getOnDiskWorkspaceFolders } from "../helpers";
import { DatabaseItem } from "../local-databases";
import { CodeQLCliServer } from "../cli";
export class DataExtensionsEditorView extends AbstractWebview<
ToDataExtensionsEditorMessage,
FromDataExtensionsEditorMessage
> {
public constructor(ctx: ExtensionContext) {
public constructor(
ctx: ExtensionContext,
private readonly cliServer: CodeQLCliServer,
private readonly queryRunner: QueryRunner,
private readonly queryStorageDir: string,
private readonly databaseItem: DatabaseItem,
) {
super(ctx);
}
@@ -49,5 +65,136 @@ export class DataExtensionsEditorView extends AbstractWebview<
protected async onWebViewLoaded() {
super.onWebViewLoaded();
await this.loadExternalApiUsages();
}
protected async loadExternalApiUsages(): Promise<void> {
const queryResult = await this.runQuery();
if (!queryResult) {
await this.clearProgress();
return;
}
await this.showProgress({
message: "Loading results",
step: 1100,
maxStep: 1500,
});
const bqrsPath = queryResult.outputDir.bqrsPath;
const results = await this.getResults(bqrsPath);
if (!results) {
await this.clearProgress();
return;
}
await this.showProgress({
message: "Finalizing results",
step: 1450,
maxStep: 1500,
});
await this.postMessage({
t: "setExternalApiRepoResults",
results,
});
await this.clearProgress();
}
private async runQuery(): Promise<CoreCompletedQuery | undefined> {
const qlpacks = await qlpackOfDatabase(this.cliServer, this.databaseItem);
const packsToSearch = [qlpacks.dbschemePack];
if (qlpacks.queryPack) {
packsToSearch.push(qlpacks.queryPack);
}
const suiteFile = (
await file({
postfix: ".qls",
})
).path;
const suiteYaml = [];
for (const qlpack of packsToSearch) {
suiteYaml.push({
from: qlpack,
queries: ".",
include: {
id: `${this.databaseItem.language}/telemetry/fetch-external-apis`,
},
});
}
await writeFile(suiteFile, dump(suiteYaml), "utf8");
const queries = await this.cliServer.resolveQueriesInSuite(
suiteFile,
getOnDiskWorkspaceFolders(),
);
if (queries.length !== 1) {
void extLogger.log(`Expected exactly one query, got ${queries.length}`);
return;
}
const query = queries[0];
const tokenSource = new CancellationTokenSource();
const queryRun = this.queryRunner.createQueryRun(
this.databaseItem.databaseUri.fsPath,
{ queryPath: query, quickEvalPosition: undefined },
false,
getOnDiskWorkspaceFolders(),
undefined,
this.queryStorageDir,
undefined,
undefined,
);
return queryRun.evaluate(
(update) => this.showProgress(update, 1500),
tokenSource.token,
new TeeLogger(this.queryRunner.logger, queryRun.outputDir.logPath),
);
}
private async getResults(bqrsPath: string) {
const bqrsInfo = await this.cliServer.bqrsInfo(bqrsPath);
if (bqrsInfo["result-sets"].length !== 1) {
void extLogger.log(
`Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`,
);
return undefined;
}
const resultSet = bqrsInfo["result-sets"][0];
await this.showProgress({
message: "Decoding results",
step: 1200,
maxStep: 1500,
});
return this.cliServer.bqrsDecode(bqrsPath, resultSet.name);
}
private async showProgress(update: ProgressUpdate, maxStep?: number) {
await this.postMessage({
t: "showProgress",
step: update.step,
maxStep: maxStep ?? update.maxStep,
message: update.message,
});
}
private async clearProgress() {
await this.showProgress({
step: 0,
maxStep: 0,
message: "",
});
}
}

View File

@@ -0,0 +1,30 @@
import { ResolvableLocationValue } from "../pure/bqrs-cli-types";
export type Call = {
label: string;
url: ResolvableLocationValue;
};
export type ExternalApiUsage = {
externalApiInfo: string;
packageName: string;
typeName: string;
methodName: string;
methodParameters: string;
supported: boolean;
usages: Call[];
};
export type ModeledMethodType =
| "none"
| "source"
| "sink"
| "summary"
| "neutral";
export type ModeledMethod = {
type: ModeledMethodType;
input: string;
output: string;
kind: string;
};

View File

@@ -861,7 +861,18 @@ async function activateWithInstalledDistribution(
);
ctx.subscriptions.push(localQueries);
const dataExtensionsEditorModule = new DataExtensionsEditorModule(ctx);
const dataExtensionsEditorQueryStorageDir = join(
tmpDir.name,
"data-extensions-editor-results",
);
await ensureDir(dataExtensionsEditorQueryStorageDir);
const dataExtensionsEditorModule = new DataExtensionsEditorModule(
ctx,
dbm,
cliServer,
qs,
dataExtensionsEditorQueryStorageDir,
);
void extLogger.log("Initializing QLTest interface.");
const testExplorerExtension = extensions.getExtension<TestHub>(

View File

@@ -5,6 +5,7 @@ import {
ResultSetSchema,
Column,
ResolvableLocationValue,
DecodedBqrsChunk,
} from "./bqrs-cli-types";
import {
VariantAnalysis,
@@ -479,6 +480,20 @@ export type ToDataFlowPathsMessage = SetDataFlowPathsMessage;
export type FromDataFlowPathsMessage = CommonFromViewMessages;
export type ToDataExtensionsEditorMessage = never;
export interface SetExternalApiResultsMessage {
t: "setExternalApiRepoResults";
results: DecodedBqrsChunk;
}
export interface ShowProgressMessage {
t: "showProgress";
step: number;
maxStep: number;
message: string;
}
export type ToDataExtensionsEditorMessage =
| SetExternalApiResultsMessage
| ShowProgressMessage;
export type FromDataExtensionsEditorMessage = ViewLoadedMsg;

View File

@@ -1,5 +1,203 @@
import * as React from "react";
import { useCallback, useEffect, useMemo, useState } from "react";
import { DecodedBqrsChunk } from "../../pure/bqrs-cli-types";
import {
ShowProgressMessage,
ToDataExtensionsEditorMessage,
} from "../../pure/interface-types";
import {
VSCodeDataGrid,
VSCodeDataGridCell,
VSCodeDataGridRow,
} from "@vscode/webview-ui-toolkit/react";
import styled from "styled-components";
import {
Call,
ExternalApiUsage,
ModeledMethod,
} from "../../data-extensions-editor/interface";
import { MethodRow } from "./MethodRow";
import { assertNever } from "../../pure/helpers-pure";
export const DataExtensionsEditorContainer = styled.div`
margin-top: 1rem;
`;
type ProgressBarProps = {
completion: number;
};
const ProgressBar = styled.div<ProgressBarProps>`
height: 10px;
width: ${(props) => props.completion * 100}%;
background-color: var(--vscode-progressBar-background);
`;
export function DataExtensionsEditor(): JSX.Element {
return <div>Data extensions editor</div>;
const [results, setResults] = useState<DecodedBqrsChunk | undefined>(
undefined,
);
const [modeledMethods, setModeledMethods] = useState<
Record<string, ModeledMethod>
>({});
const [progress, setProgress] = useState<Omit<ShowProgressMessage, "t">>({
step: 0,
maxStep: 0,
message: "",
});
useEffect(() => {
const listener = (evt: MessageEvent) => {
if (evt.origin === window.origin) {
const msg: ToDataExtensionsEditorMessage = evt.data;
switch (msg.t) {
case "setExternalApiRepoResults":
setResults(msg.results);
break;
case "showProgress":
setProgress(msg);
break;
default:
assertNever(msg);
}
} else {
// sanitize origin
const origin = evt.origin.replace(/\n|\r/g, "");
console.error(`Invalid event origin ${origin}`);
}
};
window.addEventListener("message", listener);
return () => {
window.removeEventListener("message", listener);
};
}, []);
const methods = useMemo(() => {
const methodsByApiName = new Map<string, ExternalApiUsage>();
results?.tuples.forEach((tuple) => {
const externalApiInfo = tuple[0] as string;
const supported = tuple[1] as boolean;
const usage = tuple[2] as Call;
const [packageWithType, methodDeclaration] = externalApiInfo.split("#");
const packageName = packageWithType.substring(
0,
packageWithType.lastIndexOf("."),
);
const typeName = packageWithType.substring(
packageWithType.lastIndexOf(".") + 1,
);
const methodName = methodDeclaration.substring(
0,
methodDeclaration.indexOf("("),
);
const methodParameters = methodDeclaration.substring(
methodDeclaration.indexOf("("),
);
if (!methodsByApiName.has(externalApiInfo)) {
methodsByApiName.set(externalApiInfo, {
externalApiInfo,
packageName,
typeName,
methodName,
methodParameters,
supported,
usages: [],
});
}
const method = methodsByApiName.get(externalApiInfo)!;
method.usages.push(usage);
});
const externalApiUsages = Array.from(methodsByApiName.values());
externalApiUsages.sort((a, b) => {
// Sort by number of usages descending
return b.usages.length - a.usages.length;
});
return externalApiUsages;
}, [results]);
const supportedPercentage = useMemo(() => {
return (methods.filter((m) => m.supported).length / methods.length) * 100;
}, [methods]);
const unsupportedPercentage = useMemo(() => {
return (methods.filter((m) => !m.supported).length / methods.length) * 100;
}, [methods]);
const onChange = useCallback(
(method: ExternalApiUsage, model: ModeledMethod) => {
setModeledMethods((oldModeledMethods) => ({
...oldModeledMethods,
[method.externalApiInfo]: model,
}));
},
[],
);
return (
<DataExtensionsEditorContainer>
{progress.maxStep > 0 && (
<p>
<ProgressBar completion={progress.step / progress.maxStep} />{" "}
{progress.message}
</p>
)}
{methods.length > 0 && (
<>
<div>
<h3>External API support stats</h3>
<ul>
<li>Supported: {supportedPercentage.toFixed(2)}%</li>
<li>Unsupported: {unsupportedPercentage.toFixed(2)}%</li>
</ul>
</div>
<div>
<h3>External API modelling</h3>
<VSCodeDataGrid>
<VSCodeDataGridRow rowType="header">
<VSCodeDataGridCell cellType="columnheader" gridColumn={1}>
Type
</VSCodeDataGridCell>
<VSCodeDataGridCell cellType="columnheader" gridColumn={2}>
Method
</VSCodeDataGridCell>
<VSCodeDataGridCell cellType="columnheader" gridColumn={3}>
Usages
</VSCodeDataGridCell>
<VSCodeDataGridCell cellType="columnheader" gridColumn={4}>
Model type
</VSCodeDataGridCell>
<VSCodeDataGridCell cellType="columnheader" gridColumn={5}>
Input
</VSCodeDataGridCell>
<VSCodeDataGridCell cellType="columnheader" gridColumn={6}>
Output
</VSCodeDataGridCell>
<VSCodeDataGridCell cellType="columnheader" gridColumn={7}>
Kind
</VSCodeDataGridCell>
</VSCodeDataGridRow>
{methods.map((method) => (
<MethodRow
key={method.externalApiInfo}
method={method}
model={modeledMethods[method.externalApiInfo]}
onChange={onChange}
/>
))}
</VSCodeDataGrid>
</div>
</>
)}
</DataExtensionsEditorContainer>
);
}

View File

@@ -0,0 +1,167 @@
import {
ExternalApiUsage,
ModeledMethod,
} from "../../data-extensions-editor/interface";
import {
VSCodeDataGridCell,
VSCodeDataGridRow,
VSCodeDropdown,
VSCodeOption,
VSCodeTextField,
} from "@vscode/webview-ui-toolkit/react";
import * as React from "react";
import { useCallback, useMemo } from "react";
import styled from "styled-components";
const Dropdown = styled(VSCodeDropdown)`
width: 100%;
`;
const TextField = styled(VSCodeTextField)`
width: 100%;
`;
type SupportedUnsupportedSpanProps = {
supported: boolean;
};
const SupportedUnsupportedSpan = styled.span<SupportedUnsupportedSpanProps>`
color: ${(props) => (props.supported ? "green" : "red")};
`;
type Props = {
method: ExternalApiUsage;
model: ModeledMethod | undefined;
onChange: (method: ExternalApiUsage, model: ModeledMethod) => void;
};
export const MethodRow = ({ method, model, onChange }: Props) => {
const argumentsList = useMemo(() => {
if (method.methodParameters === "()") {
return [];
}
return method.methodParameters
.substring(1, method.methodParameters.length - 1)
.split(",");
}, [method.methodParameters]);
const handleTypeInput = useCallback(
(e: InputEvent) => {
const target = e.target as HTMLSelectElement;
onChange(method, {
input: argumentsList.length === 0 ? "Argument[-1]" : "Argument[0]",
output: "ReturnType",
kind: "value",
...model,
type: target.value as ModeledMethod["type"],
});
},
[onChange, method, model, argumentsList],
);
const handleInputInput = useCallback(
(e: InputEvent) => {
if (!model) {
return;
}
const target = e.target as HTMLSelectElement;
onChange(method, {
...model,
input: target.value as ModeledMethod["input"],
});
},
[onChange, method, model],
);
const handleOutputInput = useCallback(
(e: InputEvent) => {
if (!model) {
return;
}
const target = e.target as HTMLSelectElement;
onChange(method, {
...model,
output: target.value as ModeledMethod["output"],
});
},
[onChange, method, model],
);
const handleKindInput = useCallback(
(e: InputEvent) => {
if (!model) {
return;
}
const target = e.target as HTMLSelectElement;
onChange(method, {
...model,
kind: target.value as ModeledMethod["kind"],
});
},
[onChange, method, model],
);
return (
<VSCodeDataGridRow>
<VSCodeDataGridCell gridColumn={1}>
<SupportedUnsupportedSpan supported={method.supported}>
{method.packageName}.{method.typeName}
</SupportedUnsupportedSpan>
</VSCodeDataGridCell>
<VSCodeDataGridCell gridColumn={2}>
<SupportedUnsupportedSpan supported={method.supported}>
{method.methodName}
{method.methodParameters}
</SupportedUnsupportedSpan>
</VSCodeDataGridCell>
<VSCodeDataGridCell gridColumn={3}>
{method.usages.length}
</VSCodeDataGridCell>
<VSCodeDataGridCell gridColumn={4}>
{(!method.supported || (model && model?.type !== "none")) && (
<Dropdown value={model?.type ?? "none"} onInput={handleTypeInput}>
<VSCodeOption value="none">Unmodelled</VSCodeOption>
<VSCodeOption value="source">Source</VSCodeOption>
<VSCodeOption value="sink">Sink</VSCodeOption>
<VSCodeOption value="summary">Flow summary</VSCodeOption>
<VSCodeOption value="neutral">Neutral</VSCodeOption>
</Dropdown>
)}
</VSCodeDataGridCell>
<VSCodeDataGridCell gridColumn={5}>
{model?.type && ["sink", "summary"].includes(model?.type) && (
<Dropdown value={model?.input} onInput={handleInputInput}>
<VSCodeOption value="Argument[-1]">Argument[-1]: this</VSCodeOption>
{argumentsList.map((argument, index) => (
<VSCodeOption key={argument} value={`Argument[${index}]`}>
Argument[{index}]: {argument}
</VSCodeOption>
))}
</Dropdown>
)}
</VSCodeDataGridCell>
<VSCodeDataGridCell gridColumn={6}>
{model?.type && ["source", "summary"].includes(model?.type) && (
<Dropdown value={model?.output} onInput={handleOutputInput}>
<VSCodeOption value="ReturnValue">ReturnValue</VSCodeOption>
<VSCodeOption value="Argument[-1]">Argument[-1]: this</VSCodeOption>
{argumentsList.map((argument, index) => (
<VSCodeOption key={argument} value={`Argument[${index}]`}>
Argument[{index}]: {argument}
</VSCodeOption>
))}
</Dropdown>
)}
</VSCodeDataGridCell>
<VSCodeDataGridCell gridColumn={7}>
{model?.type && ["source", "sink", "summary"].includes(model?.type) && (
<TextField value={model?.kind} onInput={handleKindInput} />
)}
</VSCodeDataGridCell>
</VSCodeDataGridRow>
);
};