Use new language definitions for reading/writing

2023-10-26 11:17:37 +02:00
parent df3b94c081
commit 8e721a6670
9 changed files with 118 additions and 55 deletions
--- a/extensions/ql-vscode/src/model-editor/auto-modeler.ts
+++ b/extensions/ql-vscode/src/model-editor/auto-modeler.ts
@@ -18,6 +18,7 @@ import { Mode } from "./shared/mode";
 import { CancellationTokenSource } from "vscode";
 import { ModelingStore } from "./modeling-store";
 import { ModelConfigListener } from "../config";
+import { QueryLanguage } from "../common/query-language";

 /**
 * The auto-modeler holds state around auto-modeling jobs and allows
@@ -36,6 +37,7 @@ export class AutoModeler {
    private readonly modelingStore: ModelingStore,
    private readonly queryStorageDir: string,
    private readonly databaseItem: DatabaseItem,
+    private readonly language: QueryLanguage,
    private readonly addModeledMethods: (
      modeledMethods: Record<string, ModeledMethod[]>,
    ) => Promise<void>,
@@ -202,7 +204,7 @@ export class AutoModeler {
      filename: "auto-model.yml",
    });

-    const loadedMethods = loadDataExtensionYaml(models);
+    const loadedMethods = loadDataExtensionYaml(models, this.language);
    if (!loadedMethods) {
      return;
    }
--- a/extensions/ql-vscode/src/model-editor/flow-model-queries.ts
+++ b/extensions/ql-vscode/src/model-editor/flow-model-queries.ts
@@ -5,7 +5,7 @@ import { QueryRunner } from "../query-server";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { showAndLogExceptionWithTelemetry } from "../common/logging";
 import { extLogger } from "../common/logging/vscode";
-import { extensiblePredicateDefinitions } from "./languages";
+import { getModelsAsDataLanguage } from "./languages";
 import { ProgressCallback } from "../common/vscode/progress";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { ModeledMethod, ModeledMethodType } from "./modeled-method";
@@ -13,12 +13,14 @@ import { redactableError } from "../common/errors";
 import { telemetryListener } from "../common/vscode/telemetry";
 import { runQuery } from "../local-queries/run-query";
 import { resolveQueries } from "../local-queries";
+import { QueryLanguage } from "../common/query-language";

 type FlowModelOptions = {
  cliServer: CodeQLCliServer;
  queryRunner: QueryRunner;
  queryStorageDir: string;
  databaseItem: DatabaseItem;
+  language: QueryLanguage;
  progress: ProgressCallback;
  token: CancellationToken;
  onResults: (results: ModeledMethod[]) => void | Promise<void>;
@@ -104,6 +106,7 @@ async function runSingleFlowQuery(
    queryRunner,
    queryStorageDir,
    databaseItem,
+    language,
    progress,
    token,
  }: Omit<FlowModelOptions, "onResults">,
@@ -140,7 +143,12 @@ async function runSingleFlowQuery(
  }

  // Interpret the results
-  const definition = extensiblePredicateDefinitions[type];
+  const modelsAsDataLanguage = getModelsAsDataLanguage(language);
+  if (!modelsAsDataLanguage) {
+    throw new Error(`No models-as-data definition for ${language}`);
+  }
+
+  const definition = modelsAsDataLanguage[type];

  const bqrsPath = completedQuery.outputDir.bqrsPath;

--- a/extensions/ql-vscode/src/model-editor/languages/index.ts
+++ b/extensions/ql-vscode/src/model-editor/languages/index.ts
@@ -1 +1,3 @@
+export * from "./languages";
+export * from "./models-as-data";
 export * from "./predicates";
--- a/extensions/ql-vscode/src/model-editor/languages/languages.ts
+++ b/extensions/ql-vscode/src/model-editor/languages/languages.ts
@@ -0,0 +1,14 @@
+import { QueryLanguage } from "../../common/query-language";
+import { ModelsAsDataLanguage } from "./models-as-data";
+import { staticLanguage } from "./static";
+
+const languages: Partial<Record<QueryLanguage, ModelsAsDataLanguage>> = {
+  [QueryLanguage.CSharp]: staticLanguage,
+  [QueryLanguage.Java]: staticLanguage,
+};
+
+export function getModelsAsDataLanguage(
+  language: QueryLanguage,
+): ModelsAsDataLanguage | undefined {
+  return languages[language];
+}
--- a/extensions/ql-vscode/src/model-editor/model-editor-view.ts
+++ b/extensions/ql-vscode/src/model-editor/model-editor-view.ts
@@ -87,6 +87,7 @@ export class ModelEditorView extends AbstractWebview<
      modelingStore,
      queryStorageDir,
      databaseItem,
+      language,
      async (modeledMethods) => {
        this.addModeledMethods(modeledMethods);
      },
@@ -223,7 +224,7 @@ export class ModelEditorView extends AbstractWebview<
              });
              await saveModeledMethods(
                this.extensionPack,
-                this.databaseItem.language,
+                this.language,
                methods,
                modeledMethods,
                mode,
@@ -399,6 +400,7 @@ export class ModelEditorView extends AbstractWebview<
    try {
      const modeledMethods = await loadModeledMethods(
        this.extensionPack,
+        this.language,
        this.cliServer,
        this.app.logger,
      );
@@ -463,6 +465,14 @@ export class ModelEditorView extends AbstractWebview<
          if (!addedDatabase) {
            return;
          }
+
+          if (addedDatabase.language !== this.language) {
+            void showAndLogErrorMessage(
+              this.app.logger,
+              `The selected database is for ${addedDatabase.language}, but the current database is for ${this.language}.`,
+            );
+            return;
+          }
        }

        progress({
@@ -477,6 +487,7 @@ export class ModelEditorView extends AbstractWebview<
            queryRunner: this.queryRunner,
            queryStorageDir: this.queryStorageDir,
            databaseItem: addedDatabase ?? this.databaseItem,
+            language: this.language,
            onResults: async (modeledMethods) => {
              const modeledMethodsByName: Record<string, ModeledMethod[]> = {};

--- a/extensions/ql-vscode/src/model-editor/modeled-method-fs.ts
+++ b/extensions/ql-vscode/src/model-editor/modeled-method-fs.ts
@@ -10,10 +10,11 @@ import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { load as loadYaml } from "js-yaml";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { pathsEqual } from "../common/files";
+import { QueryLanguage } from "../common/query-language";

 export async function saveModeledMethods(
  extensionPack: ExtensionPack,
-  language: string,
+  language: QueryLanguage,
  methods: readonly Method[],
  modeledMethods: Readonly<Record<string, readonly ModeledMethod[]>>,
  mode: Mode,
@@ -22,6 +23,7 @@ export async function saveModeledMethods(
 ): Promise<void> {
  const existingModeledMethods = await loadModeledMethodFiles(
    extensionPack,
+    language,
    cliServer,
    logger,
  );
@@ -43,6 +45,7 @@ export async function saveModeledMethods(

 async function loadModeledMethodFiles(
  extensionPack: ExtensionPack,
+  language: QueryLanguage,
  cliServer: CodeQLCliServer,
  logger: NotificationLogger,
 ): Promise<Record<string, Record<string, ModeledMethod[]>>> {
@@ -60,7 +63,7 @@ async function loadModeledMethodFiles(
      filename: modelFile,
    });

-    const modeledMethods = loadDataExtensionYaml(data);
+    const modeledMethods = loadDataExtensionYaml(data, language);
    if (!modeledMethods) {
      void showAndLogErrorMessage(
        logger,
@@ -76,6 +79,7 @@ async function loadModeledMethodFiles(

 export async function loadModeledMethods(
  extensionPack: ExtensionPack,
+  language: QueryLanguage,
  cliServer: CodeQLCliServer,
  logger: NotificationLogger,
 ): Promise<Record<string, ModeledMethod[]>> {
@@ -83,6 +87,7 @@ export async function loadModeledMethods(

  const modeledMethodsByFile = await loadModeledMethodFiles(
    extensionPack,
+    language,
    cliServer,
    logger,
  );
--- a/extensions/ql-vscode/src/model-editor/yaml.ts
+++ b/extensions/ql-vscode/src/model-editor/yaml.ts
@@ -3,21 +3,22 @@ import Ajv from "ajv";
 import { Method } from "./method";
 import { ModeledMethod, ModeledMethodType } from "./modeled-method";
 import {
-  ExtensiblePredicateDefinition,
-  extensiblePredicateDefinitions,
+  getModelsAsDataLanguage,
+  ModelsAsDataLanguageModel,
 } from "./languages";

 import * as modelExtensionFileSchema from "./model-extension-file.schema.json";
 import { Mode } from "./shared/mode";
 import { assertNever } from "../common/helpers-pure";
 import { ModelExtensionFile } from "./model-extension-file";
+import { QueryLanguage } from "../common/query-language";

 const ajv = new Ajv({ allErrors: true, allowUnionTypes: true });
 const modelExtensionFileSchemaValidate = ajv.compile(modelExtensionFileSchema);

 function createDataProperty(
  methods: readonly ModeledMethod[],
-  definition: ExtensiblePredicateDefinition,
+  definition: ModelsAsDataLanguageModel,
 ) {
  if (methods.length === 0) {
    return " []";
@@ -34,9 +35,14 @@ function createDataProperty(
 }

 export function createDataExtensionYaml(
-  language: string,
+  language: QueryLanguage,
  modeledMethods: readonly ModeledMethod[],
 ) {
+  const modelsAsDataLanguage = getModelsAsDataLanguage(language);
+  if (!modelsAsDataLanguage) {
+    throw new Error(`No models as data language for ${language}`);
+  }
+
  const methodsByType: Record<
    Exclude<ModeledMethodType, "none">,
    ModeledMethod[]
@@ -53,7 +59,7 @@ export function createDataExtensionYaml(
    }
  }

-  const extensions = Object.entries(extensiblePredicateDefinitions).map(
+  const extensions = Object.entries(modelsAsDataLanguage).map(
    ([type, definition]) => `  - addsTo:
      pack: codeql/${language}-all
      extensible: ${definition.extensiblePredicate}
@@ -69,7 +75,7 @@ ${extensions.join("\n")}`;
 }

 export function createDataExtensionYamls(
-  language: string,
+  language: QueryLanguage,
  methods: readonly Method[],
  newModeledMethods: Readonly<Record<string, readonly ModeledMethod[]>>,
  existingModeledMethods: Readonly<
@@ -98,7 +104,7 @@ export function createDataExtensionYamls(
 }

 function createDataExtensionYamlsByGrouping(
-  language: string,
+  language: QueryLanguage,
  methods: readonly Method[],
  newModeledMethods: Readonly<Record<string, readonly ModeledMethod[]>>,
  existingModeledMethods: Readonly<
@@ -153,7 +159,7 @@ function createDataExtensionYamlsByGrouping(
 }

 export function createDataExtensionYamlsForApplicationMode(
-  language: string,
+  language: QueryLanguage,
  methods: readonly Method[],
  newModeledMethods: Readonly<Record<string, readonly ModeledMethod[]>>,
  existingModeledMethods: Readonly<
@@ -170,7 +176,7 @@ export function createDataExtensionYamlsForApplicationMode(
 }

 export function createDataExtensionYamlsForFrameworkMode(
-  language: string,
+  language: QueryLanguage,
  methods: readonly Method[],
  newModeledMethods: Readonly<Record<string, readonly ModeledMethod[]>>,
  existingModeledMethods: Readonly<
@@ -240,11 +246,17 @@ function validateModelExtensionFile(data: unknown): data is ModelExtensionFile {

 export function loadDataExtensionYaml(
  data: unknown,
+  language: QueryLanguage,
 ): Record<string, ModeledMethod[]> | undefined {
  if (!validateModelExtensionFile(data)) {
    return undefined;
  }

+  const modelsAsDataLanguage = getModelsAsDataLanguage(language);
+  if (!modelsAsDataLanguage) {
+    throw new Error(`No models as data language for ${language}`);
+  }
+
  const extensions = data.extensions;

  const modeledMethods: Record<string, ModeledMethod[]> = {};
@@ -254,7 +266,7 @@ export function loadDataExtensionYaml(
    const extensible = addsTo.extensible;
    const data = extension.data;

-    const definition = Object.values(extensiblePredicateDefinitions).find(
+    const definition = Object.values(modelsAsDataLanguage).find(
      (definition) => definition.extensiblePredicate === extensible,
    );
    if (!definition) {
--- a/extensions/ql-vscode/test/unit-tests/model-editor/yaml.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/model-editor/yaml.test.ts
@@ -7,10 +7,11 @@ import {
  loadDataExtensionYaml,
 } from "../../../src/model-editor/yaml";
 import { CallClassification } from "../../../src/model-editor/method";
+import { QueryLanguage } from "../../../src/common/query-language";

 describe("createDataExtensionYaml", () => {
  it("creates the correct YAML file", () => {
-    const yaml = createDataExtensionYaml("java", [
+    const yaml = createDataExtensionYaml(QueryLanguage.Java, [
      {
        type: "sink",
        input: "Argument[0]",
@@ -50,7 +51,7 @@ describe("createDataExtensionYaml", () => {
  });

  it("includes the correct language", () => {
-    const yaml = createDataExtensionYaml("csharp", []);
+    const yaml = createDataExtensionYaml(QueryLanguage.CSharp, []);

    expect(yaml).toEqual(`extensions:
  - addsTo:
@@ -79,7 +80,7 @@ describe("createDataExtensionYaml", () => {
 describe("createDataExtensionYamlsForApplicationMode", () => {
  it("creates the correct YAML files when there are no existing modeled methods", () => {
    const yaml = createDataExtensionYamlsForApplicationMode(
-      "java",
+      QueryLanguage.Java,
      [
        {
          library: "sql2o",
@@ -323,7 +324,7 @@ describe("createDataExtensionYamlsForApplicationMode", () => {

  it("creates the correct YAML files when there are existing modeled methods", () => {
    const yaml = createDataExtensionYamlsForApplicationMode(
-      "java",
+      QueryLanguage.Java,
      [
        {
          library: "sql2o",
@@ -618,7 +619,7 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
 describe("createDataExtensionYamlsForFrameworkMode", () => {
  it("creates the correct YAML files when there are no existing modeled methods", () => {
    const yaml = createDataExtensionYamlsForFrameworkMode(
-      "java",
+      QueryLanguage.Java,
      [
        {
          library: "sql2o",
@@ -774,7 +775,7 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {

  it("creates the correct YAML files when there are existing modeled methods", () => {
    const yaml = createDataExtensionYamlsForFrameworkMode(
-      "java",
+      QueryLanguage.Java,
      [
        {
          library: "sql2o",
@@ -980,38 +981,41 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {

 describe("loadDataExtensionYaml", () => {
  it("loads the YAML file", () => {
-    const data = loadDataExtensionYaml({
-      extensions: [
-        {
-          addsTo: { pack: "codeql/java-all", extensible: "sourceModel" },
-          data: [],
-        },
-        {
-          addsTo: { pack: "codeql/java-all", extensible: "sinkModel" },
-          data: [
-            [
-              "org.sql2o",
-              "Connection",
-              true,
-              "createQuery",
-              "(String)",
-              "",
-              "Argument[0]",
-              "sql",
-              "manual",
+    const data = loadDataExtensionYaml(
+      {
+        extensions: [
+          {
+            addsTo: { pack: "codeql/java-all", extensible: "sourceModel" },
+            data: [],
+          },
+          {
+            addsTo: { pack: "codeql/java-all", extensible: "sinkModel" },
+            data: [
+              [
+                "org.sql2o",
+                "Connection",
+                true,
+                "createQuery",
+                "(String)",
+                "",
+                "Argument[0]",
+                "sql",
+                "manual",
+              ],
            ],
-          ],
-        },
-        {
-          addsTo: { pack: "codeql/java-all", extensible: "summaryModel" },
-          data: [],
-        },
-        {
-          addsTo: { pack: "codeql/java-all", extensible: "neutralModel" },
-          data: [],
-        },
-      ],
-    });
+          },
+          {
+            addsTo: { pack: "codeql/java-all", extensible: "summaryModel" },
+            data: [],
+          },
+          {
+            addsTo: { pack: "codeql/java-all", extensible: "neutralModel" },
+            data: [],
+          },
+        ],
+      },
+      QueryLanguage.Java,
+    );

    expect(data).toEqual({
      "org.sql2o.Connection#createQuery(String)": [
@@ -1033,13 +1037,16 @@ describe("loadDataExtensionYaml", () => {

  it("returns undefined if given a string", () => {
    expect(() =>
-      loadDataExtensionYaml(`extensions:
+      loadDataExtensionYaml(
+        `extensions:
  - addsTo:
      pack: codeql/java-all
      extensible: sinkModel
    data:
      - ["org.sql2o","Connection",true,"createQuery","(String)","","Argument[0]","sql","manual"]
-`),
+`,
+        QueryLanguage.Java,
+      ),
    ).toThrow("Invalid data extension YAML:  must be object");
  });
 });
--- a/extensions/ql-vscode/test/vscode-tests/cli-integration/model-editor/modeled-method-fs.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/model-editor/modeled-method-fs.test.ts
@@ -11,6 +11,7 @@ import { ExtensionPack } from "../../../../src/model-editor/shared/extension-pac
 import { join } from "path";
 import { extLogger } from "../../../../src/common/logging/vscode";
 import { homedir } from "os";
+import { QueryLanguage } from "../../../../src/common/query-language";

 const dummyExtensionPackContents = `
 name: dummy/pack
@@ -192,6 +193,7 @@ describe("modeled-method-fs", () => {

      const modeledMethods = await loadModeledMethods(
        makeExtensionPack(extensionPackPath),
+        QueryLanguage.Java,
        cli,
        extLogger,
      );