From 22024462fb3aaa71b3f51a26cb1dab857a63a994 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Fri, 23 Feb 2024 15:37:58 +0100 Subject: [PATCH] Generate separate file for generated type models in Ruby --- .../ql-vscode/src/model-editor/generate.ts | 12 +-- .../model-editor/languages/models-as-data.ts | 41 +++++++--- .../model-editor/languages/ruby/generate.ts | 16 +++- .../src/model-editor/languages/ruby/index.ts | 49 +++++++----- .../src/model-editor/model-editor-view.ts | 74 ++++++++++++++----- .../src/model-editor/modeled-method-fs.ts | 8 ++ extensions/ql-vscode/src/model-editor/yaml.ts | 2 +- .../languages/ruby/generate.test.ts | 5 ++ .../model-editor/generate.test.ts | 22 ++++-- 9 files changed, 161 insertions(+), 68 deletions(-) diff --git a/extensions/ql-vscode/src/model-editor/generate.ts b/extensions/ql-vscode/src/model-editor/generate.ts index 8f1147e65..9f4b20c13 100644 --- a/extensions/ql-vscode/src/model-editor/generate.ts +++ b/extensions/ql-vscode/src/model-editor/generate.ts @@ -5,19 +5,15 @@ import type { QueryRunner } from "../query-server"; import type { CodeQLCliServer } from "../codeql-cli/cli"; import type { ProgressCallback } from "../common/vscode/progress"; import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders"; -import type { ModeledMethod } from "./modeled-method"; import { runQuery } from "../local-queries/run-query"; import type { QueryConstraints } from "../local-queries"; import { resolveQueries } from "../local-queries"; import type { DecodedBqrs } from "../common/bqrs-cli-types"; + type GenerateQueriesOptions = { queryConstraints: QueryConstraints; filterQueries?: (queryPath: string) => boolean; - parseResults: ( - queryPath: string, - results: DecodedBqrs, - ) => ModeledMethod[] | Promise; - onResults: (results: ModeledMethod[]) => void | Promise; + onResults: (queryPath: string, results: DecodedBqrs) => void | Promise; cliServer: CodeQLCliServer; queryRunner: QueryRunner; @@ -28,7 +24,7 @@ type GenerateQueriesOptions = { }; export async function runGenerateQueries(options: GenerateQueriesOptions) { - const { queryConstraints, filterQueries, parseResults, onResults } = options; + const { queryConstraints, filterQueries, onResults } = options; options.progress({ message: "Resolving queries", @@ -55,7 +51,7 @@ export async function runGenerateQueries(options: GenerateQueriesOptions) { const bqrs = await runSingleGenerateQuery(queryPath, i, maxStep, options); if (bqrs) { - await onResults(await parseResults(queryPath, bqrs)); + await onResults(queryPath, bqrs); } } } diff --git a/extensions/ql-vscode/src/model-editor/languages/models-as-data.ts b/extensions/ql-vscode/src/model-editor/languages/models-as-data.ts index c4f6c6a57..0ea8ecea1 100644 --- a/extensions/ql-vscode/src/model-editor/languages/models-as-data.ts +++ b/extensions/ql-vscode/src/model-editor/languages/models-as-data.ts @@ -7,7 +7,7 @@ import type { SummaryModeledMethod, TypeModeledMethod, } from "../modeled-method"; -import type { DataTuple } from "../model-extension-file"; +import type { DataTuple, ModelExtension } from "../model-extension-file"; import type { Mode } from "../shared/mode"; import type { QueryConstraints } from "../../local-queries/query-constraints"; import type { @@ -32,6 +32,11 @@ export type ModelsAsDataLanguagePredicate = { readModeledMethod: ReadModeledMethod; }; +export type GenerationContext = { + mode: Mode; + isCanary: boolean; +}; + type ParseGenerationResults = ( // The path to the query that generated the results. queryPath: string, @@ -42,24 +47,37 @@ type ParseGenerationResults = ( modelsAsDataLanguage: ModelsAsDataLanguage, // The logger to use for logging. logger: BaseLogger, + // Context about this invocation of the generation. + context: GenerationContext, ) => ModeledMethod[]; type ModelsAsDataLanguageModelGeneration = { queryConstraints: (mode: Mode) => QueryConstraints; filterQueries?: (queryPath: string) => boolean; parseResults: ParseGenerationResults; +}; + +type ParseResultsToYaml = ( + // The path to the query that generated the results. + queryPath: string, + // The results of the query. + bqrs: DecodedBqrs, + // The language-specific predicate that was used to generate the results. This is passed to allow + // sharing of code between different languages. + modelsAsDataLanguage: ModelsAsDataLanguage, + // The logger to use for logging. + logger: BaseLogger, +) => ModelExtension[]; + +type ModelsAsDataLanguageAutoModelGeneration = { + queryConstraints: (mode: Mode) => QueryConstraints; + filterQueries?: (queryPath: string) => boolean; + parseResultsToYaml: ParseResultsToYaml; /** - * If autoRun is not undefined, the query will be run automatically when the user starts the - * model editor. - * - * This only applies to framework mode. Application mode will never run the query automatically. + * By default, auto model generation is enabled for all modes. This function can be used to + * override that behavior. */ - autoRun?: { - /** - * If defined, will use a custom parsing function when the query is run automatically. - */ - parseResults?: ParseGenerationResults; - }; + enabled?: (context: GenerationContext) => boolean; }; type ModelsAsDataLanguageAccessPathSuggestions = { @@ -109,6 +127,7 @@ export type ModelsAsDataLanguage = { ) => EndpointType | undefined; predicates: ModelsAsDataLanguagePredicates; modelGeneration?: ModelsAsDataLanguageModelGeneration; + autoModelGeneration?: ModelsAsDataLanguageAutoModelGeneration; accessPathSuggestions?: ModelsAsDataLanguageAccessPathSuggestions; /** * Returns the list of valid arguments that can be selected for the given method. diff --git a/extensions/ql-vscode/src/model-editor/languages/ruby/generate.ts b/extensions/ql-vscode/src/model-editor/languages/ruby/generate.ts index 64a10f0ee..3d6e3ae0c 100644 --- a/extensions/ql-vscode/src/model-editor/languages/ruby/generate.ts +++ b/extensions/ql-vscode/src/model-editor/languages/ruby/generate.ts @@ -1,6 +1,9 @@ import type { BaseLogger } from "../../../common/logging"; import type { DecodedBqrs } from "../../../common/bqrs-cli-types"; -import type { ModelsAsDataLanguage } from "../models-as-data"; +import type { + GenerationContext, + ModelsAsDataLanguage, +} from "../models-as-data"; import type { ModeledMethod } from "../../modeled-method"; import type { DataTuple } from "../../model-extension-file"; @@ -9,10 +12,21 @@ export function parseGenerateModelResults( bqrs: DecodedBqrs, modelsAsDataLanguage: ModelsAsDataLanguage, logger: BaseLogger, + { isCanary }: GenerationContext, ): ModeledMethod[] { const modeledMethods: ModeledMethod[] = []; for (const resultSetName in bqrs) { + if ( + resultSetName === + modelsAsDataLanguage.predicates.type?.extensiblePredicate && + !isCanary + ) { + // Don't load generated type results in non-canary mode. These are already automatically + // generated on start-up. + continue; + } + const definition = Object.values(modelsAsDataLanguage.predicates).find( (definition) => definition.extensiblePredicate === resultSetName, ); diff --git a/extensions/ql-vscode/src/model-editor/languages/ruby/index.ts b/extensions/ql-vscode/src/model-editor/languages/ruby/index.ts index bd6c9dc11..aa694cac4 100644 --- a/extensions/ql-vscode/src/model-editor/languages/ruby/index.ts +++ b/extensions/ql-vscode/src/model-editor/languages/ruby/index.ts @@ -177,28 +177,39 @@ export const ruby: ModelsAsDataLanguage = { "tags contain all": ["modeleditor", "generate-model", modeTag(mode)], }), parseResults: parseGenerateModelResults, - autoRun: { - parseResults: (queryPath, bqrs, modelsAsDataLanguage, logger) => { - // Only type models are generated automatically - const typePredicate = modelsAsDataLanguage.predicates.type; - if (!typePredicate) { - throw new Error("Type predicate not found"); - } + }, + autoModelGeneration: { + queryConstraints: (mode) => ({ + kind: "table", + "tags contain all": ["modeleditor", "generate-model", modeTag(mode)], + }), + parseResultsToYaml: (_queryPath, bqrs, modelsAsDataLanguage) => { + const typePredicate = modelsAsDataLanguage.predicates.type; + if (!typePredicate) { + throw new Error("Type predicate not found"); + } - const filteredBqrs = Object.fromEntries( - Object.entries(bqrs).filter( - ([key]) => key === typePredicate.extensiblePredicate, - ), - ); + const typeTuples = bqrs[typePredicate.extensiblePredicate]; + if (!typeTuples) { + return []; + } - return parseGenerateModelResults( - queryPath, - filteredBqrs, - modelsAsDataLanguage, - logger, - ); - }, + return [ + { + addsTo: { + pack: "codeql/ruby-all", + extensible: typePredicate.extensiblePredicate, + }, + data: typeTuples.tuples.filter((tuple): tuple is string[] => { + return ( + tuple.filter((x) => typeof x === "string").length === tuple.length + ); + }), + }, + ]; }, + // Only enabled for framework mode in non-canary + enabled: ({ mode, isCanary }) => mode === Mode.Framework && !isCanary, }, accessPathSuggestions: { queryConstraints: (mode) => ({ diff --git a/extensions/ql-vscode/src/model-editor/model-editor-view.ts b/extensions/ql-vscode/src/model-editor/model-editor-view.ts index 31ca255f6..3b2959ecf 100644 --- a/extensions/ql-vscode/src/model-editor/model-editor-view.ts +++ b/extensions/ql-vscode/src/model-editor/model-editor-view.ts @@ -40,8 +40,13 @@ import type { Method } from "./method"; import type { ModeledMethod } from "./modeled-method"; import type { ExtensionPack } from "./shared/extension-pack"; import type { ModelConfigListener } from "../config"; +import { isCanary } from "../config"; import { Mode } from "./shared/mode"; -import { loadModeledMethods, saveModeledMethods } from "./modeled-method-fs"; +import { + GENERATED_MODELS_SUFFIX, + loadModeledMethods, + saveModeledMethods, +} from "./modeled-method-fs"; import { pickExtensionPack } from "./extension-pack-picker"; import type { QueryLanguage } from "../common/query-language"; import { getLanguageDisplayName } from "../common/query-language"; @@ -60,6 +65,10 @@ import { parseAccessPathSuggestionRowsToOptions } from "./suggestions-bqrs"; import { ModelEvaluator } from "./model-evaluator"; import type { ModelEvaluationRunState } from "./shared/model-evaluation-run-state"; import type { VariantAnalysisManager } from "../variant-analysis/variant-analysis-manager"; +import type { ModelExtensionFile } from "./model-extension-file"; +import { modelExtensionFileToYaml } from "./yaml"; +import { outputFile } from "fs-extra"; +import { join } from "path"; export class ModelEditorView extends AbstractWebview< ToModelEditorMessage, @@ -645,14 +654,18 @@ export class ModelEditorView extends AbstractWebview< await runGenerateQueries({ queryConstraints: modelGeneration.queryConstraints(mode), filterQueries: modelGeneration.filterQueries, - parseResults: (queryPath, results) => - modelGeneration.parseResults( + onResults: async (queryPath, results) => { + const modeledMethods = modelGeneration.parseResults( queryPath, results, modelsAsDataLanguage, this.app.logger, - ), - onResults: async (modeledMethods) => { + { + mode, + isCanary: isCanary(), + }, + ); + this.addModeledMethodsFromArray(modeledMethods); }, cliServer: this.cliServer, @@ -678,15 +691,17 @@ export class ModelEditorView extends AbstractWebview< protected async generateModeledMethodsOnStartup(): Promise { const mode = this.modelingStore.getMode(this.databaseItem); - if (mode !== Mode.Framework) { + const modelsAsDataLanguage = getModelsAsDataLanguage(this.language); + const autoModelGeneration = modelsAsDataLanguage.autoModelGeneration; + + if (autoModelGeneration === undefined) { return; } - const modelsAsDataLanguage = getModelsAsDataLanguage(this.language); - const modelGeneration = modelsAsDataLanguage.modelGeneration; - const autoRun = modelGeneration?.autoRun; - - if (modelGeneration === undefined || autoRun === undefined) { + if ( + autoModelGeneration.enabled && + !autoModelGeneration.enabled({ mode, isCanary: isCanary() }) + ) { return; } @@ -698,22 +713,23 @@ export class ModelEditorView extends AbstractWebview< message: "Generating models", }); - const parseResults = - autoRun.parseResults ?? modelGeneration.parseResults; + const extensionFile: ModelExtensionFile = { + extensions: [], + }; try { await runGenerateQueries({ - queryConstraints: modelGeneration.queryConstraints(mode), - filterQueries: modelGeneration.filterQueries, - parseResults: (queryPath, results) => - parseResults( + queryConstraints: autoModelGeneration.queryConstraints(mode), + filterQueries: autoModelGeneration.filterQueries, + onResults: (queryPath, results) => { + const extensions = autoModelGeneration.parseResultsToYaml( queryPath, results, modelsAsDataLanguage, this.app.logger, - ), - onResults: async (modeledMethods) => { - this.addModeledMethodsFromArray(modeledMethods); + ); + + extensionFile.extensions.push(...extensions); }, cliServer: this.cliServer, queryRunner: this.queryRunner, @@ -730,7 +746,25 @@ export class ModelEditorView extends AbstractWebview< asError(e), )`Failed to auto-run generating models: ${getErrorMessage(e)}`, ); + return; } + + progress({ + step: 4000, + maxStep: 4000, + message: "Saving generated models", + }); + + const fileContents = `# This file was automatically generated based from ${this.databaseItem.name}. Manual changes will not persist.\n\n${modelExtensionFileToYaml(extensionFile)}`; + const filePath = join( + this.extensionPack.path, + "models", + `${this.language}${GENERATED_MODELS_SUFFIX}`, + ); + + await outputFile(filePath, fileContents); + + void this.app.logger.log(`Saved generated model file to ${filePath}`); }, { cancellable: false, diff --git a/extensions/ql-vscode/src/model-editor/modeled-method-fs.ts b/extensions/ql-vscode/src/model-editor/modeled-method-fs.ts index c7f40cd1a..f56aed39f 100644 --- a/extensions/ql-vscode/src/model-editor/modeled-method-fs.ts +++ b/extensions/ql-vscode/src/model-editor/modeled-method-fs.ts @@ -12,6 +12,9 @@ import { load as loadYaml } from "js-yaml"; import type { CodeQLCliServer } from "../codeql-cli/cli"; import { pathsEqual } from "../common/files"; import type { QueryLanguage } from "../common/query-language"; +import { isCanary } from "../config"; + +export const GENERATED_MODELS_SUFFIX = ".model.generated.yml"; export async function saveModeledMethods( extensionPack: ExtensionPack, @@ -118,6 +121,11 @@ export async function listModelFiles( for (const [path, extensions] of Object.entries(result.data)) { if (pathsEqual(path, extensionPackPath)) { for (const extension of extensions) { + // We only load generated models in canary mode + if (!isCanary() && extension.file.endsWith(GENERATED_MODELS_SUFFIX)) { + continue; + } + modelFiles.add(relative(extensionPackPath, extension.file)); } } diff --git a/extensions/ql-vscode/src/model-editor/yaml.ts b/extensions/ql-vscode/src/model-editor/yaml.ts index 10c1c9b87..a31df81cd 100644 --- a/extensions/ql-vscode/src/model-editor/yaml.ts +++ b/extensions/ql-vscode/src/model-editor/yaml.ts @@ -337,7 +337,7 @@ function validateModelExtensionFile(data: unknown): data is ModelExtensionFile { * * @param data The data extension file */ -function modelExtensionFileToYaml(data: ModelExtensionFile) { +export function modelExtensionFileToYaml(data: ModelExtensionFile) { const extensions = data.extensions .map((extension) => { const data = diff --git a/extensions/ql-vscode/test/unit-tests/model-editor/languages/ruby/generate.test.ts b/extensions/ql-vscode/test/unit-tests/model-editor/languages/ruby/generate.test.ts index fb7737bfc..c280da68b 100644 --- a/extensions/ql-vscode/test/unit-tests/model-editor/languages/ruby/generate.test.ts +++ b/extensions/ql-vscode/test/unit-tests/model-editor/languages/ruby/generate.test.ts @@ -4,6 +4,7 @@ import { ruby } from "../../../../../src/model-editor/languages/ruby"; import { createMockLogger } from "../../../../__mocks__/loggerMock"; import type { ModeledMethod } from "../../../../../src/model-editor/modeled-method"; import { EndpointType } from "../../../../../src/model-editor/method"; +import { Mode } from "../../../../../src/model-editor/shared/mode"; describe("parseGenerateModelResults", () => { it("should return the results", async () => { @@ -76,6 +77,10 @@ describe("parseGenerateModelResults", () => { bqrs, ruby, createMockLogger(), + { + isCanary: true, + mode: Mode.Framework, + }, ); expect(result).toEqual([ { diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/model-editor/generate.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/model-editor/generate.test.ts index ba79d4d29..4ac7a5945 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/model-editor/generate.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/model-editor/generate.test.ts @@ -128,14 +128,20 @@ describe("runGenerateQueries", () => { await runGenerateQueries({ queryConstraints: modelGeneration.queryConstraints(Mode.Framework), filterQueries: modelGeneration.filterQueries, - parseResults: (queryPath, results) => - modelGeneration.parseResults( - queryPath, - results, - modelsAsDataLanguage, - createMockLogger(), - ), - onResults, + onResults: (queryPath, results) => { + onResults( + modelGeneration.parseResults( + queryPath, + results, + modelsAsDataLanguage, + createMockLogger(), + { + isCanary: true, + mode: Mode.Framework, + }, + ), + ); + }, ...options, }); expect(onResults).toHaveBeenCalledWith([