mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #3043 from github/koesie10/generate-model-unify

Browse Source 
Unify model generation query execution
This commit is contained in:
Koen Vlaswinkel
2023-11-03 11:27:36 +01:00
committed by GitHub
parent eaf3a1ce1b 2f9f2f3d39
commit 4673bf56bd
14 changed files with 445 additions and 461 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
extensions/ql-vscode/src/local-queries/index.ts
View File

@@ -1,5 +1,6 @@
export * from "./local-queries";
export * from "./local-query-run";
export * from "./query-constraints";
export * from "./query-resolver";
export * from "./quick-eval-code-lens-provider";
export * from "./quick-query";

7
extensions/ql-vscode/src/local-queries/query-constraints.ts Normal file
View File

@@ -0,0 +1,7 @@
export interface QueryConstraints {
kind?: string;
"tags contain"?: string[];
"tags contain all"?: string[];
"query filename"?: string;
"query path"?: string;
}

9
extensions/ql-vscode/src/local-queries/query-resolver.ts
View File

@@ -14,6 +14,7 @@ import { showAndLogExceptionWithTelemetry } from "../common/logging";
import { extLogger } from "../common/logging/vscode";
import { telemetryListener } from "../common/vscode/telemetry";
import { SuiteInstruction } from "../packaging/suite-instruction";
import { QueryConstraints } from "./query-constraints";
export async function qlpackOfDatabase(
cli: Pick<CodeQLCliServer, "resolveQlpacks">,
@@ -27,14 +28,6 @@ export async function qlpackOfDatabase(
return await getQlPackForDbscheme(cli, dbscheme);
}
export interface QueryConstraints {
kind?: string;
"tags contain"?: string[];
"tags contain all"?: string[];
"query filename"?: string;
"query path"?: string;
}
/**
* Finds the queries with the specified kind and tags in a list of CodeQL packs.
*

195
extensions/ql-vscode/src/model-editor/flow-model-queries.ts
View File

@@ -1,195 +0,0 @@
import { CancellationToken } from "vscode";
import { DatabaseItem } from "../databases/local-databases";
import { basename } from "path";
import { QueryRunner } from "../query-server";
import { CodeQLCliServer } from "../codeql-cli/cli";
import {
NotificationLogger,
showAndLogExceptionWithTelemetry,
} from "../common/logging";
import {
getModelsAsDataLanguageModel,
ModelsAsDataLanguagePredicates,
} from "./languages";
import { ProgressCallback } from "../common/vscode/progress";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { ModeledMethod } from "./modeled-method";
import { redactableError } from "../common/errors";
import { telemetryListener } from "../common/vscode/telemetry";
import { runQuery } from "../local-queries/run-query";
import { resolveQueries } from "../local-queries";
import { QueryLanguage } from "../common/query-language";
const FLOW_MODEL_SUPPORTED_LANGUAGES = [
QueryLanguage.CSharp,
QueryLanguage.Java,
];
export function isFlowModelGenerationSupported(
language: QueryLanguage,
): boolean {
return FLOW_MODEL_SUPPORTED_LANGUAGES.includes(language);
}
type FlowModelOptions = {
cliServer: CodeQLCliServer;
queryRunner: QueryRunner;
logger: NotificationLogger;
queryStorageDir: string;
databaseItem: DatabaseItem;
language: QueryLanguage;
progress: ProgressCallback;
token: CancellationToken;
onResults: (results: ModeledMethod[]) => void | Promise<void>;
};
export async function runFlowModelQueries({
onResults,
...options
}: FlowModelOptions) {
const queries = await resolveFlowQueries(
options.cliServer,
options.databaseItem,
);
const queriesByBasename: Record<string, string> = {};
for (const query of queries) {
queriesByBasename[basename(query)] = query;
}
const summaryResults = await runSingleFlowQuery(
"summary",
queriesByBasename["CaptureSummaryModels.ql"],
0,
options,
);
if (summaryResults) {
await onResults(summaryResults);
}
const sinkResults = await runSingleFlowQuery(
"sink",
queriesByBasename["CaptureSinkModels.ql"],
1,
options,
);
if (sinkResults) {
await onResults(sinkResults);
}
const sourceResults = await runSingleFlowQuery(
"source",
queriesByBasename["CaptureSourceModels.ql"],
2,
options,
);
if (sourceResults) {
await onResults(sourceResults);
}
const neutralResults = await runSingleFlowQuery(
"neutral",
queriesByBasename["CaptureNeutralModels.ql"],
3,
options,
);
if (neutralResults) {
await onResults(neutralResults);
}
}
async function resolveFlowQueries(
cliServer: CodeQLCliServer,
databaseItem: DatabaseItem,
): Promise<string[]> {
const packsToSearch = [`codeql/${databaseItem.language}-queries`];
return await resolveQueries(
cliServer,
packsToSearch,
"flow model generator",
{
"tags contain": ["modelgenerator"],
},
);
}
async function runSingleFlowQuery(
type: keyof ModelsAsDataLanguagePredicates,
queryPath: string | undefined,
queryStep: number,
{
cliServer,
queryRunner,
logger,
queryStorageDir,
databaseItem,
language,
progress,
token,
}: Omit<FlowModelOptions, "onResults">,
): Promise<ModeledMethod[]> {
// Check that the right query was found
if (queryPath === undefined) {
void showAndLogExceptionWithTelemetry(
logger,
telemetryListener,
redactableError`Failed to find ${type} query`,
);
return [];
}
// Run the query
const completedQuery = await runQuery({
queryRunner,
databaseItem,
queryPath,
queryStorageDir,
additionalPacks: getOnDiskWorkspaceFolders(),
extensionPacks: undefined,
progress: ({ step, message }) =>
progress({
message: `Generating ${type} model: ${message}`,
step: queryStep * 1000 + step,
maxStep: 4000,
}),
token,
});
if (!completedQuery) {
return [];
}
// Interpret the results
const definition = getModelsAsDataLanguageModel(language, type);
const bqrsPath = completedQuery.outputDir.bqrsPath;
const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
if (bqrsInfo["result-sets"].length !== 1) {
void showAndLogExceptionWithTelemetry(
logger,
telemetryListener,
redactableError`Expected exactly one result set, got ${
bqrsInfo["result-sets"].length
} for ${basename(queryPath)}`,
);
}
const resultSet = bqrsInfo["result-sets"][0];
const decodedResults = await cliServer.bqrsDecode(bqrsPath, resultSet.name);
const results = decodedResults.tuples;
return (
results
// This is just a sanity check. The query should only return strings.
.filter((result) => typeof result[0] === "string")
.map((result) => {
const row = result[0] as string;
return definition.readModeledMethod(row.split(";"));
})
);
}

162
extensions/ql-vscode/src/model-editor/generate-model-queries.ts
View File

@@ -1,162 +0,0 @@
import { CancellationToken } from "vscode";
import { DatabaseItem } from "../databases/local-databases";
import { QueryRunner } from "../query-server";
import { CodeQLCliServer } from "../codeql-cli/cli";
import {
NotificationLogger,
showAndLogExceptionWithTelemetry,
} from "../common/logging";
import { getModelsAsDataLanguage } from "./languages";
import { ProgressCallback } from "../common/vscode/progress";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { ModeledMethod } from "./modeled-method";
import { redactableError } from "../common/errors";
import { telemetryListener } from "../common/vscode/telemetry";
import { runQuery } from "../local-queries/run-query";
import { resolveQueries } from "../local-queries";
import { QueryLanguage } from "../common/query-language";
import { DataTuple } from "./model-extension-file";
const GENERATE_MODEL_SUPPORTED_LANGUAGES = [QueryLanguage.Ruby];
export function isGenerateModelSupported(language: QueryLanguage): boolean {
return GENERATE_MODEL_SUPPORTED_LANGUAGES.includes(language);
}
type GenerateModelOptions = {
cliServer: CodeQLCliServer;
queryRunner: QueryRunner;
logger: NotificationLogger;
queryStorageDir: string;
databaseItem: DatabaseItem;
language: QueryLanguage;
progress: ProgressCallback;
token: CancellationToken;
};
// resolve (100) + query (1000) + interpret (100)
const maxStep = 1200;
export async function runGenerateModelQuery({
cliServer,
queryRunner,
logger,
queryStorageDir,
databaseItem,
language,
progress,
token,
}: GenerateModelOptions): Promise<ModeledMethod[]> {
progress({
message: "Resolving generate model query",
step: 100,
maxStep,
});
const queryPath = await resolveGenerateModelQuery(
cliServer,
logger,
databaseItem,
);
if (queryPath === undefined) {
return [];
}
// Run the query
const completedQuery = await runQuery({
queryRunner,
databaseItem,
queryPath,
queryStorageDir,
additionalPacks: getOnDiskWorkspaceFolders(),
extensionPacks: undefined,
progress: ({ step, message }) =>
progress({
message: `Generating models: ${message}`,
step: 100 + step,
maxStep,
}),
token,
});
if (!completedQuery) {
return [];
}
progress({
message: "Decoding results",
step: 1100,
maxStep,
});
const decodedBqrs = await cliServer.bqrsDecodeAll(
completedQuery.outputDir.bqrsPath,
);
const modelsAsDataLanguage = getModelsAsDataLanguage(language);
const modeledMethods: ModeledMethod[] = [];
for (const resultSetName in decodedBqrs) {
const definition = Object.values(modelsAsDataLanguage.predicates).find(
(definition) => definition.extensiblePredicate === resultSetName,
);
if (definition === undefined) {
void logger.log(`No predicate found for ${resultSetName}`);
continue;
}
const resultSet = decodedBqrs[resultSetName];
if (
resultSet.tuples.some((tuple) =>
tuple.some((value) => typeof value === "object"),
)
) {
void logger.log(
`Skipping ${resultSetName} because it contains undefined values`,
);
continue;
}
modeledMethods.push(
...resultSet.tuples.map((tuple) => {
const row = tuple.filter(
(value): value is DataTuple => typeof value !== "object",
);
return definition.readModeledMethod(row);
}),
);
}
return modeledMethods;
}
async function resolveGenerateModelQuery(
cliServer: CodeQLCliServer,
logger: NotificationLogger,
databaseItem: DatabaseItem,
): Promise<string | undefined> {
const packsToSearch = [`codeql/${databaseItem.language}-queries`];
const queries = await resolveQueries(
cliServer,
packsToSearch,
"generate model",
{
"query path": "queries/modeling/GenerateModel.ql",
},
);
if (queries.length !== 1) {
void showAndLogExceptionWithTelemetry(
logger,
telemetryListener,
redactableError`Expected exactly one generate model query, got ${queries.length}`,
);
return undefined;
}
return queries[0];
}

99
extensions/ql-vscode/src/model-editor/generate.ts Normal file
View File

@@ -0,0 +1,99 @@
import { CancellationToken } from "vscode";
import { DatabaseItem } from "../databases/local-databases";
import { basename } from "path";
import { QueryRunner } from "../query-server";
import { CodeQLCliServer } from "../codeql-cli/cli";
import { ProgressCallback } from "../common/vscode/progress";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { ModeledMethod } from "./modeled-method";
import { runQuery } from "../local-queries/run-query";
import { QueryConstraints, resolveQueries } from "../local-queries";
import { DecodedBqrs } from "../common/bqrs-cli-types";
type GenerateQueriesOptions = {
queryConstraints: QueryConstraints;
filterQueries?: (queryPath: string) => boolean;
parseResults: (
queryPath: string,
results: DecodedBqrs,
) => ModeledMethod[] | Promise<ModeledMethod[]>;
onResults: (results: ModeledMethod[]) => void | Promise<void>;
cliServer: CodeQLCliServer;
queryRunner: QueryRunner;
queryStorageDir: string;
databaseItem: DatabaseItem;
progress: ProgressCallback;
token: CancellationToken;
};
export async function runGenerateQueries(options: GenerateQueriesOptions) {
const { queryConstraints, filterQueries, parseResults, onResults } = options;
options.progress({
message: "Resolving queries",
step: 1,
maxStep: 5000,
});
const packsToSearch = [`codeql/${options.databaseItem.language}-queries`];
const queryPaths = await resolveQueries(
options.cliServer,
packsToSearch,
"generate model",
queryConstraints,
);
const filteredQueryPaths = filterQueries
? queryPaths.filter(filterQueries)
: queryPaths;
const maxStep = filteredQueryPaths.length * 1000;
for (let i = 0; i < filteredQueryPaths.length; i++) {
const queryPath = filteredQueryPaths[i];
const bqrs = await runSingleGenerateQuery(queryPath, i, maxStep, options);
if (bqrs) {
await onResults(await parseResults(queryPath, bqrs));
}
}
}
async function runSingleGenerateQuery(
queryPath: string,
queryStep: number,
maxStep: number,
{
cliServer,
queryRunner,
queryStorageDir,
databaseItem,
progress,
token,
}: GenerateQueriesOptions,
): Promise<DecodedBqrs | undefined> {
const queryBasename = basename(queryPath);
// Run the query
const completedQuery = await runQuery({
queryRunner,
databaseItem,
queryPath,
queryStorageDir,
additionalPacks: getOnDiskWorkspaceFolders(),
extensionPacks: undefined,
progress: ({ step, message }) =>
progress({
message: `Generating model from ${queryBasename}: ${message}`,
step: queryStep * 1000 + step,
maxStep,
}),
token,
});
if (!completedQuery) {
return undefined;
}
return cliServer.bqrsDecodeAll(completedQuery.outputDir.bqrsPath);
}

20
extensions/ql-vscode/src/model-editor/languages/models-as-data.ts
View File

@@ -8,6 +8,9 @@ import {
} from "../modeled-method";
import { DataTuple } from "../model-extension-file";
import { Mode } from "../shared/mode";
import type { QueryConstraints } from "../../local-queries/query-constraints";
import { DecodedBqrs } from "../../common/bqrs-cli-types";
import { BaseLogger } from "../../common/logging";
type GenerateMethodDefinition<T> = (method: T) => DataTuple[];
type ReadModeledMethod = (row: DataTuple[]) => ModeledMethod;
@@ -19,6 +22,22 @@ export type ModelsAsDataLanguagePredicate<T> = {
readModeledMethod: ReadModeledMethod;
};
type ModelsAsDataLanguageModelGeneration = {
queryConstraints: QueryConstraints;
filterQueries?: (queryPath: string) => boolean;
parseResults: (
// The path to the query that generated the results.
queryPath: string,
// The results of the query.
bqrs: DecodedBqrs,
// The language-specific predicate that was used to generate the results. This is passed to allow
// sharing of code between different languages.
modelsAsDataLanguage: ModelsAsDataLanguage,
// The logger to use for logging.
logger: BaseLogger,
) => ModeledMethod[];
};
export type ModelsAsDataLanguagePredicates = {
source?: ModelsAsDataLanguagePredicate<SourceModeledMethod>;
sink?: ModelsAsDataLanguagePredicate<SinkModeledMethod>;
@@ -34,4 +53,5 @@ export type ModelsAsDataLanguage = {
availableModes?: Mode[];
createMethodSignature: (method: MethodDefinition) => string;
predicates: ModelsAsDataLanguagePredicates;
modelGeneration?: ModelsAsDataLanguageModelGeneration;
};

50
extensions/ql-vscode/src/model-editor/languages/ruby/generate.ts Normal file
View File

@@ -0,0 +1,50 @@
import { BaseLogger } from "../../../common/logging";
import { DecodedBqrs } from "../../../common/bqrs-cli-types";
import { ModelsAsDataLanguage } from "../models-as-data";
import { ModeledMethod } from "../../modeled-method";
import { DataTuple } from "../../model-extension-file";
export function parseGenerateModelResults(
_queryPath: string,
bqrs: DecodedBqrs,
modelsAsDataLanguage: ModelsAsDataLanguage,
logger: BaseLogger,
): ModeledMethod[] {
const modeledMethods: ModeledMethod[] = [];
for (const resultSetName in bqrs) {
const definition = Object.values(modelsAsDataLanguage.predicates).find(
(definition) => definition.extensiblePredicate === resultSetName,
);
if (definition === undefined) {
void logger.log(`No predicate found for ${resultSetName}`);
continue;
}
const resultSet = bqrs[resultSetName];
if (
resultSet.tuples.some((tuple) =>
tuple.some((value) => typeof value === "object"),
)
) {
void logger.log(
`Skipping ${resultSetName} because it contains undefined values`,
);
continue;
}
modeledMethods.push(
...resultSet.tuples.map((tuple) => {
const row = tuple.filter(
(value): value is DataTuple => typeof value !== "object",
);
return definition.readModeledMethod(row);
}),
);
}
return modeledMethods;
}

13
extensions/ql-vscode/src/model-editor/languages/ruby.ts → extensions/ql-vscode/src/model-editor/languages/ruby/index.ts
View File

@@ -1,6 +1,7 @@
import { ModelsAsDataLanguage } from "./models-as-data";
import { sharedExtensiblePredicates, sharedKinds } from "./shared";
import { Mode } from "../shared/mode";
import { ModelsAsDataLanguage } from "../models-as-data";
import { sharedExtensiblePredicates, sharedKinds } from "../shared";
import { Mode } from "../../shared/mode";
import { parseGenerateModelResults } from "./generate";
function parseRubyMethodFromPath(path: string): string {
const match = path.match(/Method\[([^\]]+)].*/);
@@ -150,4 +151,10 @@ export const ruby: ModelsAsDataLanguage = {
},
},
},
modelGeneration: {
queryConstraints: {
"query path": "queries/modeling/GenerateModel.ql",
},
parseResults: parseGenerateModelResults,
},
};

60
extensions/ql-vscode/src/model-editor/languages/static/generate.ts Normal file
View File

@@ -0,0 +1,60 @@
import { BaseLogger } from "../../../common/logging";
import {
ModelsAsDataLanguage,
ModelsAsDataLanguagePredicates,
} from "../models-as-data";
import { DecodedBqrs } from "../../../common/bqrs-cli-types";
import { ModeledMethod } from "../../modeled-method";
import { basename } from "../../../common/path";
const queriesToModel: Record<string, keyof ModelsAsDataLanguagePredicates> = {
"CaptureSummaryModels.ql": "summary",
"CaptureSinkModels.ql": "sink",
"CaptureSourceModels.ql": "source",
"CaptureNeutralModels.ql": "neutral",
};
export function filterFlowModelQueries(queryPath: string): boolean {
return Object.keys(queriesToModel).includes(basename(queryPath));
}
export function parseFlowModelResults(
queryPath: string,
bqrs: DecodedBqrs,
modelsAsDataLanguage: ModelsAsDataLanguage,
logger: BaseLogger,
): ModeledMethod[] {
if (Object.keys(bqrs).length !== 1) {
throw new Error(
`Expected exactly one result set from ${queryPath}, but got ${
Object.keys(bqrs).length
}`,
);
}
const modelType = queriesToModel[basename(queryPath)];
if (!modelType) {
void logger.log(`Unknown model type for ${queryPath}`);
return [];
}
const resultSet = bqrs[Object.keys(bqrs)[0]];
const results = resultSet.tuples;
const definition = modelsAsDataLanguage.predicates[modelType];
if (!definition) {
throw new Error(`No definition for ${modelType}`);
}
return (
results
// This is just a sanity check. The query should only return strings.
.filter((result) => typeof result[0] === "string")
.map((result) => {
const row = result[0] as string;
return definition.readModeledMethod(row.split(";"));
})
);
}

16
extensions/ql-vscode/src/model-editor/languages/static.ts → extensions/ql-vscode/src/model-editor/languages/static/index.ts
View File

@@ -1,7 +1,8 @@
import { ModelsAsDataLanguage } from "./models-as-data";
import { Provenance } from "../modeled-method";
import { DataTuple } from "../model-extension-file";
import { sharedExtensiblePredicates, sharedKinds } from "./shared";
import { ModelsAsDataLanguage } from "../models-as-data";
import { Provenance } from "../../modeled-method";
import { DataTuple } from "../../model-extension-file";
import { sharedExtensiblePredicates, sharedKinds } from "../shared";
import { filterFlowModelQueries, parseFlowModelResults } from "./generate";
function readRowToMethod(row: DataTuple[]): string {
return `${row[0]}.${row[1]}#${row[3]}${row[4]}`;
@@ -137,4 +138,11 @@ export const staticLanguage: ModelsAsDataLanguage = {
}),
},
},
modelGeneration: {
queryConstraints: {
"tags contain": ["modelgenerator"],
},
filterQueries: filterFlowModelQueries,
parseResults: parseFlowModelResults,
},
};

81
extensions/ql-vscode/src/model-editor/model-editor-view.ts
View File

@@ -23,10 +23,6 @@ import {
import { DatabaseItem, DatabaseManager } from "../databases/local-databases";
import { CodeQLCliServer } from "../codeql-cli/cli";
import { asError, assertNever, getErrorMessage } from "../common/helpers-pure";
import {
isFlowModelGenerationSupported,
runFlowModelQueries,
} from "./flow-model-queries";
import { promptImportGithubDatabase } from "../databases/database-fetcher";
import { App } from "../common/app";
import { redactableError } from "../common/errors";
@@ -51,10 +47,7 @@ import { ModelingStore } from "./modeling-store";
import { ModelEditorViewTracker } from "./model-editor-view-tracker";
import { ModelingEvents } from "./modeling-events";
import { getModelsAsDataLanguage, ModelsAsDataLanguage } from "./languages";
import {
isGenerateModelSupported,
runGenerateModelQuery,
} from "./generate-model-queries";
import { runGenerateQueries } from "./generate";
export class ModelEditorView extends AbstractWebview<
ToModelEditorMessage,
@@ -270,11 +263,8 @@ export class ModelEditorView extends AbstractWebview<
break;
case "generateMethod":
if (isFlowModelGenerationSupported(this.language)) {
await this.generateModeledMethodsFromFlow();
} else if (isGenerateModelSupported(this.language)) {
await this.generateModeledMethodsFromGenerateModel();
}
await this.generateModeledMethods();
void telemetryListener?.sendUIInteraction(
"model-editor-generate-modeled-methods",
);
@@ -377,10 +367,10 @@ export class ModelEditorView extends AbstractWebview<
}
private async setViewState(): Promise<void> {
const modelsAsDataLanguage = getModelsAsDataLanguage(this.language);
const showGenerateButton =
this.modelConfig.flowGeneration &&
(isFlowModelGenerationSupported(this.language) ||
isGenerateModelSupported(this.language));
this.modelConfig.flowGeneration && !!modelsAsDataLanguage.modelGeneration;
const showLlmButton =
this.databaseItem.language === "java" && this.modelConfig.llmGeneration;
@@ -474,13 +464,23 @@ export class ModelEditorView extends AbstractWebview<
}
}
protected async generateModeledMethodsFromFlow(): Promise<void> {
protected async generateModeledMethods(): Promise<void> {
await withProgress(
async (progress) => {
const tokenSource = new CancellationTokenSource();
const mode = this.modelingStore.getMode(this.databaseItem);
const modelsAsDataLanguage = getModelsAsDataLanguage(this.language);
const modelGeneration = modelsAsDataLanguage.modelGeneration;
if (!modelGeneration) {
void showAndLogErrorMessage(
this.app.logger,
`Model generation is not supported for ${this.language}.`,
);
return;
}
let addedDatabase: DatabaseItem | undefined;
// In application mode, we need the database of a specific library to generate
@@ -509,51 +509,26 @@ export class ModelEditorView extends AbstractWebview<
});
try {
await runFlowModelQueries({
cliServer: this.cliServer,
queryRunner: this.queryRunner,
logger: this.app.logger,
queryStorageDir: this.queryStorageDir,
databaseItem: addedDatabase ?? this.databaseItem,
language: this.language,
await runGenerateQueries({
queryConstraints: modelGeneration.queryConstraints,
filterQueries: modelGeneration.filterQueries,
parseResults: (queryPath, results) =>
modelGeneration.parseResults(
queryPath,
results,
modelsAsDataLanguage,
this.app.logger,
),
onResults: async (modeledMethods) => {
this.addModeledMethodsFromArray(modeledMethods);
},
progress,
token: tokenSource.token,
});
} catch (e: unknown) {
void showAndLogExceptionWithTelemetry(
this.app.logger,
this.app.telemetry,
redactableError(
asError(e),
)`Failed to generate flow model: ${getErrorMessage(e)}`,
);
}
},
{ cancellable: false },
);
}
protected async generateModeledMethodsFromGenerateModel(): Promise<void> {
await withProgress(
async (progress) => {
const tokenSource = new CancellationTokenSource();
try {
const modeledMethods = await runGenerateModelQuery({
cliServer: this.cliServer,
queryRunner: this.queryRunner,
logger: this.app.logger,
queryStorageDir: this.queryStorageDir,
databaseItem: this.databaseItem,
language: this.language,
databaseItem: addedDatabase ?? this.databaseItem,
progress,
token: tokenSource.token,
});
this.addModeledMethodsFromArray(modeledMethods);
} catch (e: unknown) {
void showAndLogExceptionWithTelemetry(
this.app.logger,

107
extensions/ql-vscode/test/unit-tests/model-editor/languages/ruby/generate.test.ts Normal file
View File

@@ -0,0 +1,107 @@
import { DecodedBqrs } from "../../../../../src/common/bqrs-cli-types";
import { parseGenerateModelResults } from "../../../../../src/model-editor/languages/ruby/generate";
import { ruby } from "../../../../../src/model-editor/languages/ruby";
import { createMockLogger } from "../../../../__mocks__/loggerMock";
describe("parseGenerateModelResults", () => {
it("should return the results", async () => {
const bqrs: DecodedBqrs = {
sourceModel: {
columns: [
{ name: "type", kind: "String" },
{ name: "path", kind: "String" },
{ name: "kind", kind: "String" },
],
tuples: [],
},
sinkModel: {
columns: [
{ name: "type", kind: "String" },
{ name: "path", kind: "String" },
{ name: "kind", kind: "String" },
],
tuples: [],
},
typeVariableModel: {
columns: [
{ name: "name", kind: "String" },
{ name: "path", kind: "String" },
],
tuples: [],
},
typeModel: {
columns: [
{ name: "type1", kind: "String" },
{ name: "type2", kind: "String" },
{ name: "path", kind: "String" },
],
tuples: [
["Array", "SQLite3::ResultSet", "Method[types].ReturnValue"],
["Array", "SQLite3::ResultSet", "Method[columns].ReturnValue"],
["Array", "SQLite3::Statement", "Method[types].ReturnValue"],
["Array", "SQLite3::Statement", "Method[columns].ReturnValue"],
],
},
summaryModel: {
columns: [
{ name: "type", kind: "String" },
{ name: "path", kind: "String" },
{ name: "input", kind: "String" },
{ name: "output", kind: "String" },
{ name: "kind", kind: "String" },
],
tuples: [
[
"SQLite3::Database",
"Method[create_function]",
"Argument[self]",
"ReturnValue",
"value",
],
[
"SQLite3::Value!",
"Method[new]",
"Argument[1]",
"ReturnValue",
"value",
],
],
},
};
const result = parseGenerateModelResults(
"/a/b/c/query.ql",
bqrs,
ruby,
createMockLogger(),
);
expect(result.sort()).toEqual(
[
{
input: "Argument[self]",
kind: "value",
methodName: "create_function",
methodParameters: "",
output: "ReturnValue",
packageName: "",
provenance: "manual",
signature: "SQLite3::Database#create_function",
type: "summary",
typeName: "SQLite3::Database",
},
{
input: "Argument[1]",
kind: "value",
methodName: "new",
methodParameters: "",
output: "ReturnValue",
packageName: "",
provenance: "manual",
signature: "SQLite3::Value!#new",
type: "summary",
typeName: "SQLite3::Value!",
},
].sort(),
);
});
});

86
extensions/ql-vscode/test/vscode-tests/no-workspace/model-editor/generate-model-queries.test.ts → extensions/ql-vscode/test/vscode-tests/no-workspace/model-editor/generate.test.ts
View File

@@ -5,23 +5,29 @@ import {
} from "../../../../src/databases/local-databases";
import { file } from "tmp-promise";
import { QueryResultType } from "../../../../src/query-server/new-messages";
import { Mode } from "../../../../src/model-editor/shared/mode";
import { mockedObject, mockedUri } from "../../utils/mocking.helpers";
import { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
import { QueryRunner } from "../../../../src/query-server";
import { join } from "path";
import { CancellationTokenSource } from "vscode-jsonrpc";
import { QueryOutputDir } from "../../../../src/run-queries-shared";
import { runGenerateModelQuery } from "../../../../src/model-editor/generate-model-queries";
import { QueryLanguage } from "../../../../src/common/query-language";
import { runGenerateQueries } from "../../../../src/model-editor/generate";
import { ruby } from "../../../../src/model-editor/languages/ruby";
describe("runGenerateQueries", () => {
const modelsAsDataLanguage = ruby;
const modelGeneration = modelsAsDataLanguage.modelGeneration;
if (!modelGeneration) {
throw new Error("Test requires a model generation step");
}
describe("runGenerateModelQuery", () => {
it("should run the query and return the results", async () => {
const queryStorageDir = (await file()).path;
const outputDir = new QueryOutputDir(join(queryStorageDir, "1"));
const onResults = jest.fn();
const options = {
mode: Mode.Application,
cliServer: mockedObject<CodeQLCliServer>({
resolveQueriesInSuite: jest
.fn()
@@ -100,7 +106,6 @@ describe("runGenerateModelQuery", () => {
}),
logger: createMockLogger(),
}),
logger: createMockLogger(),
databaseItem: mockedObject<DatabaseItem>({
databaseUri: mockedUri("/a/b/c/src.zip"),
contents: {
@@ -114,41 +119,50 @@ describe("runGenerateModelQuery", () => {
.mockResolvedValue("/home/runner/work/my-repo/my-repo"),
sourceArchive: mockedUri("/a/b/c/src.zip"),
}),
language: QueryLanguage.Ruby,
queryStorageDir: "/tmp/queries",
progress: jest.fn(),
token: new CancellationTokenSource().token,
};
const result = await runGenerateModelQuery(options);
expect(result.sort()).toEqual(
[
{
input: "Argument[self]",
kind: "value",
methodName: "create_function",
methodParameters: "",
output: "ReturnValue",
packageName: "",
provenance: "manual",
signature: "SQLite3::Database#create_function",
type: "summary",
typeName: "SQLite3::Database",
},
{
input: "Argument[1]",
kind: "value",
methodName: "new",
methodParameters: "",
output: "ReturnValue",
packageName: "",
provenance: "manual",
signature: "SQLite3::Value!#new",
type: "summary",
typeName: "SQLite3::Value!",
},
].sort(),
);
await runGenerateQueries({
queryConstraints: modelGeneration.queryConstraints,
filterQueries: modelGeneration.filterQueries,
parseResults: (queryPath, results) =>
modelGeneration.parseResults(
queryPath,
results,
modelsAsDataLanguage,
createMockLogger(),
),
onResults,
...options,
});
expect(onResults).toHaveBeenCalledWith([
{
input: "Argument[self]",
kind: "value",
methodName: "create_function",
methodParameters: "",
output: "ReturnValue",
packageName: "",
provenance: "manual",
signature: "SQLite3::Database#create_function",
type: "summary",
typeName: "SQLite3::Database",
},
{
input: "Argument[1]",
kind: "value",
methodName: "new",
methodParameters: "",
output: "ReturnValue",
packageName: "",
provenance: "manual",
signature: "SQLite3::Value!#new",
type: "summary",
typeName: "SQLite3::Value!",
},
]);
expect(options.queryRunner.createQueryRun).toHaveBeenCalledTimes(1);
expect(options.queryRunner.createQueryRun).toHaveBeenCalledWith(