Merge pull request #2264 from github/koesie10/data-extension-editor-yaml

Add saving of data extension editor table to YAML

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts

@@ -1,4 +1,10 @@
-import { CancellationTokenSource, ExtensionContext, ViewColumn } from "vscode";
+import {
+  CancellationTokenSource,
+  ExtensionContext,
+  Uri,
+  ViewColumn,
+  workspace,
+} from "vscode";
 import { AbstractWebview, WebviewPanelConfig } from "../abstract-webview";
 import {
   FromDataExtensionsEditorMessage,
@@ -17,9 +23,9 @@ import {
 } from "../helpers";
 import { DatabaseItem } from "../local-databases";
 import { CodeQLCliServer } from "../cli";
+import { assertNever, asError, getErrorMessage } from "../pure/helpers-pure";
 import { decodeBqrsToExternalApiUsages } from "./bqrs";
 import { redactableError } from "../pure/errors";
-import { asError, getErrorMessage } from "../pure/helpers-pure";
 
 export class DataExtensionsEditorView extends AbstractWebview<
   ToDataExtensionsEditorMessage,
@@ -63,9 +69,14 @@ export class DataExtensionsEditorView extends AbstractWebview<
       case "viewLoaded":
         await this.onWebViewLoaded();
 
+        break;
+      case "applyDataExtensionYaml":
+        await this.saveYaml(msg.yaml);
+        await this.loadExternalApiUsages();
+
         break;
       default:
-        throw new Error("Unexpected message type");
+        assertNever(msg);
     }
   }
 
@@ -75,6 +86,17 @@ export class DataExtensionsEditorView extends AbstractWebview<
     await this.loadExternalApiUsages();
   }
 
+  protected async saveYaml(yaml: string): Promise<void> {
+    const modelFilename = this.calculateModelFilename();
+    if (!modelFilename) {
+      return;
+    }
+
+    await writeFile(modelFilename, yaml);
+
+    void extLogger.log(`Saved data extension YAML to ${modelFilename}`);
+  }
+
   protected async loadExternalApiUsages(): Promise<void> {
     try {
       const queryResult = await this.runQuery();
@@ -221,4 +243,21 @@ export class DataExtensionsEditorView extends AbstractWebview<
       message: "",
     });
   }
+
+  private calculateModelFilename(): string | undefined {
+    const workspaceFolder = workspace.workspaceFolders?.find(
+      (folder) => folder.name === "ql",
+    );
+    if (!workspaceFolder) {
+      void extLogger.log("No workspace folder 'ql' found");
+
+      return;
+    }
+
+    return Uri.joinPath(
+      workspaceFolder.uri,
+      "java/ql/lib/ext",
+      `${this.databaseItem.name.replaceAll("/", ".")}.model.yml`,
+    ).fsPath;
+  }
 }

extensions/ql-vscode/src/data-extensions-editor/yaml.ts

@@ -0,0 +1,144 @@
+import { ExternalApiUsage } from "./external-api-usage";
+import { ModeledMethod, ModeledMethodType } from "./modeled-method";
+
+type ExternalApiUsageByType = {
+  externalApiUsage: ExternalApiUsage;
+  modeledMethod: ModeledMethod;
+};
+
+type DataExtensionDefinition = {
+  extensible: string;
+  generateMethodDefinition: (method: ExternalApiUsageByType) => any[];
+};
+
+const definitions: Record<
+  Exclude<ModeledMethodType, "none">,
+  DataExtensionDefinition
+> = {
+  source: {
+    extensible: "sourceModel",
+    // extensible predicate sourceModel(
+    //   string package, string type, boolean subtypes, string name, string signature, string ext,
+    //   string output, string kind, string provenance
+    // );
+    generateMethodDefinition: (method) => [
+      method.externalApiUsage.packageName,
+      method.externalApiUsage.typeName,
+      true,
+      method.externalApiUsage.methodName,
+      method.externalApiUsage.methodParameters,
+      "",
+      method.modeledMethod.output,
+      method.modeledMethod.kind,
+      "manual",
+    ],
+  },
+  sink: {
+    extensible: "sinkModel",
+    // extensible predicate sinkModel(
+    //   string package, string type, boolean subtypes, string name, string signature, string ext,
+    //   string input, string kind, string provenance
+    // );
+    generateMethodDefinition: (method) => [
+      method.externalApiUsage.packageName,
+      method.externalApiUsage.typeName,
+      true,
+      method.externalApiUsage.methodName,
+      method.externalApiUsage.methodParameters,
+      "",
+      method.modeledMethod.input,
+      method.modeledMethod.kind,
+      "manual",
+    ],
+  },
+  summary: {
+    extensible: "summaryModel",
+    // extensible predicate summaryModel(
+    //   string package, string type, boolean subtypes, string name, string signature, string ext,
+    //   string input, string output, string kind, string provenance
+    // );
+    generateMethodDefinition: (method) => [
+      method.externalApiUsage.packageName,
+      method.externalApiUsage.typeName,
+      true,
+      method.externalApiUsage.methodName,
+      method.externalApiUsage.methodParameters,
+      "",
+      method.modeledMethod.input,
+      method.modeledMethod.output,
+      method.modeledMethod.kind,
+      "manual",
+    ],
+  },
+  neutral: {
+    extensible: "neutralModel",
+    // extensible predicate neutralModel(
+    //   string package, string type, string name, string signature, string provenance
+    // );
+    generateMethodDefinition: (method) => [
+      method.externalApiUsage.packageName,
+      method.externalApiUsage.typeName,
+      method.externalApiUsage.methodName,
+      method.externalApiUsage.methodParameters,
+      "manual",
+    ],
+  },
+};
+
+function createDataProperty(
+  methods: ExternalApiUsageByType[],
+  definition: DataExtensionDefinition,
+) {
+  if (methods.length === 0) {
+    return " []";
+  }
+
+  return `\n${methods
+    .map(
+      (method) =>
+        `      - ${JSON.stringify(
+          definition.generateMethodDefinition(method),
+        )}`,
+    )
+    .join("\n")}`;
+}
+
+export function createDataExtensionYaml(
+  externalApiUsages: ExternalApiUsage[],
+  modeledMethods: Record<string, ModeledMethod>,
+) {
+  const methodsByType: Record<
+    Exclude<ModeledMethodType, "none">,
+    ExternalApiUsageByType[]
+  > = {
+    source: [],
+    sink: [],
+    summary: [],
+    neutral: [],
+  };
+
+  for (const externalApiUsage of externalApiUsages) {
+    const modeledMethod = modeledMethods[externalApiUsage.signature];
+
+    if (modeledMethod?.type && modeledMethod.type !== "none") {
+      methodsByType[modeledMethod.type].push({
+        externalApiUsage,
+        modeledMethod,
+      });
+    }
+  }
+
+  const extensions = Object.entries(definitions).map(
+    ([type, definition]) => `  - addsTo:
+      pack: codeql/java-all
+      extensible: ${definition.extensible}
+    data:${createDataProperty(
+      methodsByType[type as Exclude<ModeledMethodType, "none">],
+      definition,
+    )}
+`,
+  );
+
+  return `extensions:
+${extensions.join("\n")}`;
+}

extensions/ql-vscode/src/pure/interface-types.ts

@@ -492,8 +492,15 @@ export interface ShowProgressMessage {
   message: string;
 }
 
+export interface ApplyDataExtensionYamlMessage {
+  t: "applyDataExtensionYaml";
+  yaml: string;
+}
+
 export type ToDataExtensionsEditorMessage =
   | SetExternalApiUsagesMessage
   | ShowProgressMessage;
 
-export type FromDataExtensionsEditorMessage = ViewLoadedMsg;
+export type FromDataExtensionsEditorMessage =
+  | ViewLoadedMsg
+  | ApplyDataExtensionYamlMessage;

extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx

@@ -5,6 +5,7 @@ import {
   ToDataExtensionsEditorMessage,
 } from "../../pure/interface-types";
 import {
+  VSCodeButton,
   VSCodeDataGrid,
   VSCodeDataGridCell,
   VSCodeDataGridRow,
@@ -14,6 +15,8 @@ import { ExternalApiUsage } from "../../data-extensions-editor/external-api-usag
 import { ModeledMethod } from "../../data-extensions-editor/modeled-method";
 import { MethodRow } from "./MethodRow";
 import { assertNever } from "../../pure/helpers-pure";
+import { vscode } from "../vscode-api";
+import { createDataExtensionYaml } from "../../data-extensions-editor/yaml";
 import { calculateSupportedPercentage } from "./supported";
 
 export const DataExtensionsEditorContainer = styled.div`
@@ -88,6 +91,18 @@ export function DataExtensionsEditor(): JSX.Element {
     [],
   );
 
+  const onApplyClick = useCallback(() => {
+    const yamlString = createDataExtensionYaml(
+      externalApiUsages,
+      modeledMethods,
+    );
+
+    vscode.postMessage({
+      t: "applyDataExtensionYaml",
+      yaml: yamlString,
+    });
+  }, [externalApiUsages, modeledMethods]);
+
   return (
     <DataExtensionsEditorContainer>
       {progress.maxStep > 0 && (
@@ -108,6 +123,7 @@ export function DataExtensionsEditor(): JSX.Element {
           </div>
           <div>
             <h3>External API modelling</h3>
+            <VSCodeButton onClick={onApplyClick}>Apply</VSCodeButton>
             <VSCodeDataGrid>
               <VSCodeDataGridRow rowType="header">
                 <VSCodeDataGridCell cellType="columnheader" gridColumn={1}>

extensions/ql-vscode/test/unit-tests/data-extensions-editor/yaml.test.ts

@@ -0,0 +1,101 @@
+import { createDataExtensionYaml } from "../../../src/data-extensions-editor/yaml";
+
+describe("createDataExtensionYaml", () => {
+  it("creates the correct YAML file", () => {
+    const yaml = createDataExtensionYaml(
+      [
+        {
+          signature: "org.sql2o.Connection#createQuery(String)",
+          packageName: "org.sql2o",
+          typeName: "Connection",
+          methodName: "createQuery",
+          methodParameters: "(String)",
+          supported: true,
+          usages: [
+            {
+              label: "createQuery(...)",
+              url: {
+                uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+                startLine: 15,
+                startColumn: 13,
+                endLine: 15,
+                endColumn: 56,
+              },
+            },
+            {
+              label: "createQuery(...)",
+              url: {
+                uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+                startLine: 26,
+                startColumn: 13,
+                endLine: 26,
+                endColumn: 39,
+              },
+            },
+          ],
+        },
+        {
+          signature: "org.sql2o.Query#executeScalar(Class)",
+          packageName: "org.sql2o",
+          typeName: "Query",
+          methodName: "executeScalar",
+          methodParameters: "(Class)",
+          supported: true,
+          usages: [
+            {
+              label: "executeScalar(...)",
+              url: {
+                uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+                startLine: 15,
+                startColumn: 13,
+                endLine: 15,
+                endColumn: 85,
+              },
+            },
+            {
+              label: "executeScalar(...)",
+              url: {
+                uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+                startLine: 26,
+                startColumn: 13,
+                endLine: 26,
+                endColumn: 68,
+              },
+            },
+          ],
+        },
+      ],
+      {
+        "org.sql2o.Connection#createQuery(String)": {
+          type: "sink",
+          input: "Argument[0]",
+          output: "",
+          kind: "sql",
+        },
+      },
+    );
+
+    expect(yaml).toEqual(`extensions:
+  - addsTo:
+      pack: codeql/java-all
+      extensible: sourceModel
+    data: []
+
+  - addsTo:
+      pack: codeql/java-all
+      extensible: sinkModel
+    data:
+      - ["org.sql2o","Connection",true,"createQuery","(String)","","Argument[0]","sql","manual"]
+
+  - addsTo:
+      pack: codeql/java-all
+      extensible: summaryModel
+    data: []
+
+  - addsTo:
+      pack: codeql/java-all
+      extensible: neutralModel
+    data: []
+`);
+  });
+});