mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #2658 from github/nora-koen/data-extensions-editor-without-ql-submodule

Browse Source 
Remove submodules dependency from data extension editor
This commit is contained in:
Nora
2023-08-02 17:30:46 +02:00
committed by GitHub
parent 1289ab509c 868fae093d
commit 80ae9a4b36
7 changed files with 342 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
View File

@@ -6,12 +6,14 @@ import { QueryRunner } from "../query-server";
import { DatabaseItem } from "../databases/local-databases";
import { interpretResultsSarif } from "../query-results";
import { ProgressCallback } from "../common/vscode/progress";
import { Mode } from "./shared/mode";
type Options = {
cliServer: CodeQLCliServer;
queryRunner: QueryRunner;
databaseItem: DatabaseItem;
queryStorageDir: string;
queryDir: string;
progress: ProgressCallback;
};
@@ -23,6 +25,7 @@ export async function getAutoModelUsages({
queryRunner,
databaseItem,
queryStorageDir,
queryDir,
progress,
}: Options): Promise<UsageSnippetsBySignature> {
const maxStep = 1500;
@@ -32,11 +35,12 @@ export async function getAutoModelUsages({
// This will re-run the query that was already run when opening the data extensions editor. This
// might be unnecessary, but this makes it really easy to get the path to the BQRS file which we
// need to interpret the results.
const queryResult = await runQuery("applicationModeQuery", {
const queryResult = await runQuery(Mode.Application, {
cliServer,
queryRunner,
queryStorageDir,
databaseItem,
queryDir,
progress: (update) =>
progress({
maxStep,

36
extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts
View File

@@ -9,7 +9,17 @@ import { join } from "path";
import { App } from "../common/app";
import { withProgress } from "../common/vscode/progress";
import { pickExtensionPack } from "./extension-pack-picker";
import { showAndLogErrorMessage } from "../common/logging";
import {
showAndLogErrorMessage,
showAndLogExceptionWithTelemetry,
} from "../common/logging";
import { dir } from "tmp-promise";
import { fetchExternalApiQueries } from "./queries";
import { telemetryListener } from "../common/vscode/telemetry";
import { redactableError } from "../common/errors";
import { extLogger } from "../common/logging/vscode";
import { isQueryLanguage } from "../common/query-language";
import { setUpPack } from "./external-api-usage-query";
const SUPPORTED_LANGUAGES: string[] = ["java", "csharp"];
@@ -60,10 +70,14 @@ export class DataExtensionsEditorModule {
return;
}
if (!SUPPORTED_LANGUAGES.includes(db.language)) {
const language = db.language;
if (
!SUPPORTED_LANGUAGES.includes(language) ||
!isQueryLanguage(language)
) {
void showAndLogErrorMessage(
this.app.logger,
`The data extensions editor is not supported for ${db.language} databases.`,
`The data extensions editor is not supported for ${language} databases.`,
);
return;
}
@@ -99,6 +113,21 @@ export class DataExtensionsEditorModule {
return;
}
const query = fetchExternalApiQueries[language];
if (!query) {
void showAndLogExceptionWithTelemetry(
extLogger,
telemetryListener,
redactableError`No external API usage query found for language ${language}`,
);
return;
}
// Create new temporary directory for query files and pack dependencies
const queryDir = (await dir({ unsafeCleanup: true })).path;
await setUpPack(queryDir, query, language);
await this.cliServer.packInstall(queryDir);
const view = new DataExtensionsEditorView(
this.ctx,
this.app,
@@ -106,6 +135,7 @@ export class DataExtensionsEditorModule {
this.cliServer,
this.queryRunner,
this.queryStorageDir,
queryDir,
db,
modelFile,
);

13
extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
View File

@@ -71,6 +71,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
private readonly cliServer: CodeQLCliServer,
private readonly queryRunner: QueryRunner,
private readonly queryStorageDir: string,
private readonly queryDir: string,
private readonly databaseItem: DatabaseItem,
private readonly extensionPack: ExtensionPack,
private mode: Mode = Mode.Application,
@@ -248,19 +249,15 @@ export class DataExtensionsEditorView extends AbstractWebview<
async (progress) => {
try {
const cancellationTokenSource = new CancellationTokenSource();
const queryResult = await runQuery(
this.mode === Mode.Framework
? "frameworkModeQuery"
: "applicationModeQuery",
{
const queryResult = await runQuery(this.mode, {
cliServer: this.cliServer,
queryRunner: this.queryRunner,
databaseItem: this.databaseItem,
queryStorageDir: this.queryStorageDir,
queryDir: this.queryDir,
progress: (update) => progress({ ...update, maxStep: 1500 }),
token: cancellationTokenSource.token,
},
);
});
if (!queryResult) {
return;
}
@@ -432,6 +429,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
cliServer: this.cliServer,
queryRunner: this.queryRunner,
queryStorageDir: this.queryStorageDir,
queryDir: this.queryDir,
databaseItem: this.databaseItem,
progress: (update) => progress({ ...update, maxStep }),
});
@@ -512,6 +510,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
this.cliServer,
this.queryRunner,
this.queryStorageDir,
this.queryDir,
addedDatabase,
modelFile,
Mode.Framework,

4
extensions/ql-vscode/src/data-extensions-editor/extension-pack-picker.ts
View File

@@ -43,6 +43,10 @@ export async function pickExtensionPack(
// Get all existing extension packs in the workspace
const additionalPacks = getOnDiskWorkspaceFolders();
// the CLI doesn't check packs in the .github folder, so we need to add it manually
if (additionalPacks.length === 1) {
additionalPacks.push(`${additionalPacks[0]}/.github`);
}
const extensionPacksInfo = await cliServer.resolveQlpacks(
additionalPacks,
true,

89
extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts
View File

@@ -1,71 +1,44 @@
import { CoreCompletedQuery, QueryRunner } from "../query-server";
import { dir } from "tmp-promise";
import { writeFile } from "fs-extra";
import { dump as dumpYaml } from "js-yaml";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { extLogger } from "../common/logging/vscode";
import { showAndLogExceptionWithTelemetry, TeeLogger } from "../common/logging";
import { isQueryLanguage } from "../common/query-language";
import { CancellationToken } from "vscode";
import { CodeQLCliServer } from "../codeql-cli/cli";
import { DatabaseItem } from "../databases/local-databases";
import { ProgressCallback } from "../common/vscode/progress";
import { fetchExternalApiQueries } from "./queries";
import { QueryResultType } from "../query-server/new-messages";
import { join } from "path";
import { redactableError } from "../common/errors";
import { telemetryListener } from "../common/vscode/telemetry";
import { join } from "path";
import { Mode } from "./shared/mode";
import { writeFile } from "fs-extra";
import { Query } from "./queries/query";
import { QueryLanguage } from "../common/query-language";
import { dump } from "js-yaml";
type RunQueryOptions = {
cliServer: Pick<CodeQLCliServer, "resolveQlpacks">;
queryRunner: Pick<QueryRunner, "createQueryRun" | "logger">;
databaseItem: Pick<DatabaseItem, "contents" | "databaseUri" | "language">;
queryStorageDir: string;
queryDir: string;
progress: ProgressCallback;
token: CancellationToken;
};
export async function runQuery(
queryName: keyof Omit<Query, "dependencies">,
{
cliServer,
queryRunner,
databaseItem,
queryStorageDir,
progress,
token,
}: RunQueryOptions,
): Promise<CoreCompletedQuery | undefined> {
// The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
// move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
// This is intentionally not pretty code, as it will be removed soon.
// For a reference of what this should do in the future, see the previous implementation in
// https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72
if (!isQueryLanguage(databaseItem.language)) {
void showAndLogExceptionWithTelemetry(
extLogger,
telemetryListener,
redactableError`Unsupported database language ${databaseItem.language}`,
export async function setUpPack(
queryDir: string,
query: Query,
language: QueryLanguage,
) {
Object.values(Mode).map(async (mode) => {
const queryFile = join(
queryDir,
`FetchExternalApis${mode.charAt(0).toUpperCase() + mode.slice(1)}Mode.ql`,
);
return;
}
const query = fetchExternalApiQueries[databaseItem.language];
if (!query) {
void showAndLogExceptionWithTelemetry(
extLogger,
telemetryListener,
redactableError`No external API usage query found for language ${databaseItem.language}`,
);
return;
}
const queryDir = (await dir({ unsafeCleanup: true })).path;
const queryFile = join(queryDir, "FetchExternalApis.ql");
await writeFile(queryFile, query[queryName], "utf8");
await writeFile(queryFile, query[`${mode}ModeQuery`], "utf8");
});
if (query.dependencies) {
for (const [filename, contents] of Object.entries(query.dependencies)) {
@@ -78,18 +51,42 @@ export async function runQuery(
name: "codeql/external-api-usage",
version: "0.0.0",
dependencies: {
[`codeql/${databaseItem.language}-all`]: "*",
[`codeql/${language}-all`]: "*",
},
};
const qlpackFile = join(queryDir, "codeql-pack.yml");
await writeFile(qlpackFile, dumpYaml(syntheticQueryPack), "utf8");
await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8");
}
export async function runQuery(
mode: Mode,
{
cliServer,
queryRunner,
databaseItem,
queryStorageDir,
queryDir,
progress,
token,
}: RunQueryOptions,
): Promise<CoreCompletedQuery | undefined> {
// The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
// move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
// This is intentionally not pretty code, as it will be removed soon.
// For a reference of what this should do in the future, see the previous implementation in
// https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72
const additionalPacks = getOnDiskWorkspaceFolders();
const extensionPacks = Object.keys(
await cliServer.resolveQlpacks(additionalPacks, true),
);
const queryFile = join(
queryDir,
`FetchExternalApis${mode.charAt(0).toUpperCase() + mode.slice(1)}Mode.ql`,
);
const queryRun = queryRunner.createQueryRun(
databaseItem.databaseUri.fsPath,
{

5
extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/extension-pack-picker.test.ts
View File

@@ -99,7 +99,10 @@ describe("pickExtensionPack", () => {
name: "codeql-custom-queries-java",
index: 0,
};
additionalPacks = [Uri.file(tmpDir).fsPath];
additionalPacks = [
Uri.file(tmpDir).fsPath,
`${Uri.file(tmpDir).fsPath}/.github`,
];
workspaceFoldersSpy = jest
.spyOn(workspace, "workspaceFolders", "get")
.mockReturnValue([workspaceFolder]);

193
extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts
View File

@@ -1,41 +1,151 @@
import {
readQueryResults,
runQuery,
setUpPack,
} from "../../../../src/data-extensions-editor/external-api-usage-query";
import { createMockLogger } from "../../../__mocks__/loggerMock";
import { DatabaseKind } from "../../../../src/databases/local-databases";
import { file } from "tmp-promise";
import { dirSync, file } from "tmp-promise";
import { QueryResultType } from "../../../../src/query-server/new-messages";
import { readdir, readFile } from "fs-extra";
import { load } from "js-yaml";
import { dirname, join } from "path";
import { fetchExternalApiQueries } from "../../../../src/data-extensions-editor/queries";
import * as log from "../../../../src/common/logging/notifications";
import { RedactableError } from "../../../../src/common/errors";
import { showAndLogExceptionWithTelemetry } from "../../../../src/common/logging";
import { QueryLanguage } from "../../../../src/common/query-language";
import { Query } from "../../../../src/data-extensions-editor/queries/query";
import { mockedUri } from "../../utils/mocking.helpers";
import { Mode } from "../../../../src/data-extensions-editor/shared/mode";
import { readFile, readFileSync, readdir } from "fs-extra";
import { join } from "path";
import { load } from "js-yaml";
describe("runQuery", () => {
const cases = Object.keys(fetchExternalApiQueries).flatMap((lang) => {
describe("external api usage query", () => {
describe("setUpPack", () => {
const languages = Object.keys(fetchExternalApiQueries).flatMap((lang) => {
const queryDir = dirSync({ unsafeCleanup: true }).name;
const query = fetchExternalApiQueries[lang as QueryLanguage];
if (!query) {
return [];
}
const keys = new Set(Object.keys(query));
keys.delete("dependencies");
return Array.from(keys).map((name) => ({
language: lang as QueryLanguage,
queryName: name as keyof Omit<Query, "dependencies">,
}));
return { language: lang as QueryLanguage, queryDir, query };
});
test.each(cases)(
"should run $queryName for $language",
async ({ language, queryName }) => {
test.each(languages)(
"should create files for $language",
async ({ language, queryDir, query }) => {
await setUpPack(queryDir, query, language);
const queryFiles = await readdir(queryDir);
expect(queryFiles.sort()).toEqual(
[
"codeql-pack.yml",
"FetchExternalApisApplicationMode.ql",
"FetchExternalApisFrameworkMode.ql",
"AutomodelVsCode.qll",
].sort(),
);
const suiteFileContents = await readFile(
join(queryDir, "codeql-pack.yml"),
"utf8",
);
const suiteYaml = load(suiteFileContents);
expect(suiteYaml).toEqual({
name: "codeql/external-api-usage",
version: "0.0.0",
dependencies: {
[`codeql/${language}-all`]: "*",
},
});
Object.values(Mode).forEach((mode) => {
expect(
readFileSync(
join(
queryDir,
`FetchExternalApis${
mode.charAt(0).toUpperCase() + mode.slice(1)
}Mode.ql`,
),
"utf8",
),
).toEqual(query[`${mode}ModeQuery`]);
});
for (const [filename, contents] of Object.entries(
query.dependencies ?? {},
)) {
expect(await readFile(join(queryDir, filename), "utf8")).toEqual(
contents,
);
}
},
);
});
describe("runQuery", () => {
const language = Object.keys(fetchExternalApiQueries)[
Math.floor(Math.random() * Object.keys(fetchExternalApiQueries).length)
] as QueryLanguage;
const queryDir = dirSync({ unsafeCleanup: true }).name;
it("should log an error", async () => {
const showAndLogExceptionWithTelemetrySpy: jest.SpiedFunction<
typeof showAndLogExceptionWithTelemetry
> = jest.spyOn(log, "showAndLogExceptionWithTelemetry");
const logPath = (await file()).path;
const query = fetchExternalApiQueries[language];
if (!query) {
throw new Error(`No query found for language ${language}`);
}
const options = {
cliServer: {
resolveQlpacks: jest.fn().mockResolvedValue({
"my/extensions": "/a/b/c/",
}),
},
queryRunner: {
createQueryRun: jest.fn().mockReturnValue({
evaluate: jest.fn().mockResolvedValue({
resultType: QueryResultType.CANCELLATION,
}),
outputDir: {
logPath,
},
}),
logger: createMockLogger(),
},
databaseItem: {
databaseUri: mockedUri("/a/b/c/src.zip"),
contents: {
kind: DatabaseKind.Database,
name: "foo",
datasetUri: mockedUri(),
},
language,
},
queryStorageDir: "/tmp/queries",
queryDir,
progress: jest.fn(),
token: {
isCancellationRequested: false,
onCancellationRequested: jest.fn(),
},
};
expect(await runQuery(Mode.Application, options)).toBeUndefined();
expect(showAndLogExceptionWithTelemetrySpy).toHaveBeenCalledWith(
expect.anything(),
undefined,
expect.any(RedactableError),
);
});
it("should run query for random language", async () => {
const logPath = (await file()).path;
const query = fetchExternalApiQueries[language];
@@ -70,6 +180,7 @@ describe("runQuery", () => {
language,
},
queryStorageDir: "/tmp/queries",
queryDir,
progress: jest.fn(),
token: {
isCancellationRequested: false,
@@ -77,7 +188,7 @@ describe("runQuery", () => {
},
};
const result = await runQuery(queryName, options);
const result = await runQuery(Mode.Framework, options);
expect(result?.resultType).toEqual(QueryResultType.SUCCESS);
@@ -86,7 +197,7 @@ describe("runQuery", () => {
expect(options.queryRunner.createQueryRun).toHaveBeenCalledWith(
"/a/b/c/src.zip",
{
queryPath: expect.stringMatching(/FetchExternalApis\.ql/),
queryPath: expect.stringMatching(/FetchExternalApis\S*\.ql/),
quickEvalPosition: undefined,
quickEvalCountOnly: false,
},
@@ -97,49 +208,10 @@ describe("runQuery", () => {
undefined,
undefined,
);
const queryPath =
options.queryRunner.createQueryRun.mock.calls[0][1].queryPath;
const queryDirectory = dirname(queryPath);
const queryFiles = await readdir(queryDirectory);
expect(queryFiles.sort()).toEqual(
[
"codeql-pack.yml",
"FetchExternalApis.ql",
"AutomodelVsCode.qll",
].sort(),
);
const suiteFileContents = await readFile(
join(queryDirectory, "codeql-pack.yml"),
"utf8",
);
const suiteYaml = load(suiteFileContents);
expect(suiteYaml).toEqual({
name: "codeql/external-api-usage",
version: "0.0.0",
dependencies: {
[`codeql/${language}-all`]: "*",
},
});
});
expect(
await readFile(join(queryDirectory, "FetchExternalApis.ql"), "utf8"),
).toEqual(query[queryName]);
for (const [filename, contents] of Object.entries(
query.dependencies ?? {},
)) {
expect(await readFile(join(queryDirectory, filename), "utf8")).toEqual(
contents,
);
}
},
);
});
describe("readQueryResults", () => {
describe("readQueryResults", () => {
const options = {
cliServer: {
bqrsInfo: jest.fn(),
@@ -256,4 +328,5 @@ describe("readQueryResults", () => {
"#select",
);
});
});
});